Compare commits
1 Commits
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | d23a8a83e5 |
|
@ -16,7 +16,6 @@ env:
|
||||||
BUILDENV_IMAGE: git.yourcmc.ru/vitalif/vitastor/buildenv
|
BUILDENV_IMAGE: git.yourcmc.ru/vitalif/vitastor/buildenv
|
||||||
TEST_IMAGE: git.yourcmc.ru/vitalif/vitastor/test
|
TEST_IMAGE: git.yourcmc.ru/vitalif/vitastor/test
|
||||||
OSD_ARGS: '--etcd_quick_timeout 2000'
|
OSD_ARGS: '--etcd_quick_timeout 2000'
|
||||||
USE_RAMDISK: 1
|
|
||||||
|
|
||||||
concurrency:
|
concurrency:
|
||||||
group: ci-${{ github.ref }}
|
group: ci-${{ github.ref }}
|
||||||
|
@ -198,24 +197,6 @@ jobs:
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
test_etcd_fail_antietcd:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
|
||||||
steps:
|
|
||||||
- name: Run test
|
|
||||||
id: test
|
|
||||||
timeout-minutes: 10
|
|
||||||
run: ANTIETCD=1 /root/vitastor/tests/test_etcd_fail.sh
|
|
||||||
- name: Print logs
|
|
||||||
if: always() && steps.test.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
|
||||||
echo "-------- $i --------"
|
|
||||||
cat $i
|
|
||||||
echo ""
|
|
||||||
done
|
|
||||||
|
|
||||||
test_interrupted_rebalance:
|
test_interrupted_rebalance:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
@ -558,24 +539,6 @@ jobs:
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
test_dd:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
|
||||||
steps:
|
|
||||||
- name: Run test
|
|
||||||
id: test
|
|
||||||
timeout-minutes: 3
|
|
||||||
run: /root/vitastor/tests/test_dd.sh
|
|
||||||
- name: Print logs
|
|
||||||
if: always() && steps.test.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
|
||||||
echo "-------- $i --------"
|
|
||||||
cat $i
|
|
||||||
echo ""
|
|
||||||
done
|
|
||||||
|
|
||||||
test_root_node:
|
test_root_node:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
@ -702,24 +665,6 @@ jobs:
|
||||||
echo ""
|
echo ""
|
||||||
done
|
done
|
||||||
|
|
||||||
test_heal_antietcd:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
needs: build
|
|
||||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
|
||||||
steps:
|
|
||||||
- name: Run test
|
|
||||||
id: test
|
|
||||||
timeout-minutes: 10
|
|
||||||
run: ANTIETCD=1 /root/vitastor/tests/test_heal.sh
|
|
||||||
- name: Print logs
|
|
||||||
if: always() && steps.test.outcome == 'failure'
|
|
||||||
run: |
|
|
||||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
|
||||||
echo "-------- $i --------"
|
|
||||||
cat $i
|
|
||||||
echo ""
|
|
||||||
done
|
|
||||||
|
|
||||||
test_heal_csum_32k_dmj:
|
test_heal_csum_32k_dmj:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
needs: build
|
needs: build
|
||||||
|
|
|
@ -34,10 +34,6 @@ for my $line (<>)
|
||||||
{
|
{
|
||||||
$test_name .= '_imm';
|
$test_name .= '_imm';
|
||||||
}
|
}
|
||||||
elsif ($1 eq 'ANTIETCD')
|
|
||||||
{
|
|
||||||
$test_name .= '_antietcd';
|
|
||||||
}
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
$test_name .= '_'.lc($1).'_'.$2;
|
$test_name .= '_'.lc($1).'_'.$2;
|
||||||
|
|
|
@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
|
||||||
|
|
||||||
project(vitastor)
|
project(vitastor)
|
||||||
|
|
||||||
set(VITASTOR_VERSION "1.9.1")
|
set(VERSION "1.6.1")
|
||||||
|
|
||||||
add_subdirectory(src)
|
add_subdirectory(src)
|
||||||
|
|
|
@ -19,7 +19,7 @@ Vitastor нацелен в первую очередь на SSD и SSD+HDD кл
|
||||||
TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
|
TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
|
||||||
что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
|
что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
|
||||||
|
|
||||||
Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, OpenNebula, Proxmox, Kubernetes.
|
Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, Proxmox, Kubernetes.
|
||||||
Другие драйверы могут также быть легко реализованы.
|
Другие драйверы могут также быть легко реализованы.
|
||||||
|
|
||||||
Подробности смотрите в документации по ссылкам ниже.
|
Подробности смотрите в документации по ссылкам ниже.
|
||||||
|
@ -42,7 +42,6 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
|
||||||
- Установка
|
- Установка
|
||||||
- [Пакеты](docs/installation/packages.ru.md)
|
- [Пакеты](docs/installation/packages.ru.md)
|
||||||
- [Proxmox](docs/installation/proxmox.ru.md)
|
- [Proxmox](docs/installation/proxmox.ru.md)
|
||||||
- [OpenNebula](docs/installation/opennebula.ru.md)
|
|
||||||
- [OpenStack](docs/installation/openstack.ru.md)
|
- [OpenStack](docs/installation/openstack.ru.md)
|
||||||
- [Kubernetes CSI](docs/installation/kubernetes.ru.md)
|
- [Kubernetes CSI](docs/installation/kubernetes.ru.md)
|
||||||
- [Сборка из исходных кодов](docs/installation/source.ru.md)
|
- [Сборка из исходных кодов](docs/installation/source.ru.md)
|
||||||
|
|
|
@ -19,7 +19,7 @@ supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1
|
||||||
with proper hardware which is ~10 times faster than other popular SDS's like Ceph
|
with proper hardware which is ~10 times faster than other popular SDS's like Ceph
|
||||||
or internal systems of public clouds.
|
or internal systems of public clouds.
|
||||||
|
|
||||||
Vitastor supports QEMU, NBD, NFS protocols, OpenStack, OpenNebula, Proxmox, Kubernetes drivers.
|
Vitastor supports QEMU, NBD, NFS protocols, OpenStack, Proxmox, Kubernetes drivers.
|
||||||
More drivers may be created easily.
|
More drivers may be created easily.
|
||||||
|
|
||||||
Read more details below in the documentation.
|
Read more details below in the documentation.
|
||||||
|
@ -42,7 +42,6 @@ Read more details below in the documentation.
|
||||||
- Installation
|
- Installation
|
||||||
- [Packages](docs/installation/packages.en.md)
|
- [Packages](docs/installation/packages.en.md)
|
||||||
- [Proxmox](docs/installation/proxmox.en.md)
|
- [Proxmox](docs/installation/proxmox.en.md)
|
||||||
- [OpenNebula](docs/installation/opennebula.en.md)
|
|
||||||
- [OpenStack](docs/installation/openstack.en.md)
|
- [OpenStack](docs/installation/openstack.en.md)
|
||||||
- [Kubernetes CSI](docs/installation/kubernetes.en.md)
|
- [Kubernetes CSI](docs/installation/kubernetes.en.md)
|
||||||
- [Building from Source](docs/installation/source.en.md)
|
- [Building from Source](docs/installation/source.en.md)
|
||||||
|
|
|
@ -1,9 +1,9 @@
|
||||||
VITASTOR_VERSION ?= v1.9.1
|
VERSION ?= v1.6.1
|
||||||
|
|
||||||
all: build push
|
all: build push
|
||||||
|
|
||||||
build:
|
build:
|
||||||
@docker build --rm -t vitalif/vitastor-csi:$(VITASTOR_VERSION) .
|
@docker build --rm -t vitalif/vitastor-csi:$(VERSION) .
|
||||||
|
|
||||||
push:
|
push:
|
||||||
@docker push vitalif/vitastor-csi:$(VITASTOR_VERSION)
|
@docker push vitalif/vitastor-csi:$(VERSION)
|
||||||
|
|
|
@ -49,7 +49,7 @@ spec:
|
||||||
capabilities:
|
capabilities:
|
||||||
add: ["SYS_ADMIN"]
|
add: ["SYS_ADMIN"]
|
||||||
allowPrivilegeEscalation: true
|
allowPrivilegeEscalation: true
|
||||||
image: vitalif/vitastor-csi:v1.9.1
|
image: vitalif/vitastor-csi:v1.6.1
|
||||||
args:
|
args:
|
||||||
- "--node=$(NODE_ID)"
|
- "--node=$(NODE_ID)"
|
||||||
- "--endpoint=$(CSI_ENDPOINT)"
|
- "--endpoint=$(CSI_ENDPOINT)"
|
||||||
|
|
|
@ -121,7 +121,7 @@ spec:
|
||||||
privileged: true
|
privileged: true
|
||||||
capabilities:
|
capabilities:
|
||||||
add: ["SYS_ADMIN"]
|
add: ["SYS_ADMIN"]
|
||||||
image: vitalif/vitastor-csi:v1.9.1
|
image: vitalif/vitastor-csi:v1.6.1
|
||||||
args:
|
args:
|
||||||
- "--node=$(NODE_ID)"
|
- "--node=$(NODE_ID)"
|
||||||
- "--endpoint=$(CSI_ENDPOINT)"
|
- "--endpoint=$(CSI_ENDPOINT)"
|
||||||
|
|
|
@ -5,7 +5,7 @@ package vitastor
|
||||||
|
|
||||||
const (
|
const (
|
||||||
vitastorCSIDriverName = "csi.vitastor.io"
|
vitastorCSIDriverName = "csi.vitastor.io"
|
||||||
vitastorCSIDriverVersion = "1.9.1"
|
vitastorCSIDriverVersion = "1.6.1"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Config struct fills the parameters of request or user input
|
// Config struct fills the parameters of request or user input
|
||||||
|
|
|
@ -1,4 +1,4 @@
|
||||||
vitastor (1.9.1-1) unstable; urgency=medium
|
vitastor (1.6.1-1) unstable; urgency=medium
|
||||||
|
|
||||||
* Bugfixes
|
* Bugfixes
|
||||||
|
|
||||||
|
|
|
@ -53,9 +53,3 @@ Architecture: amd64
|
||||||
Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version})
|
Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version})
|
||||||
Description: Vitastor Proxmox Virtual Environment storage plugin
|
Description: Vitastor Proxmox Virtual Environment storage plugin
|
||||||
Vitastor storage plugin for Proxmox Virtual Environment.
|
Vitastor storage plugin for Proxmox Virtual Environment.
|
||||||
|
|
||||||
Package: vitastor-opennebula
|
|
||||||
Architecture: amd64
|
|
||||||
Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client, patch, python3, jq
|
|
||||||
Description: Vitastor OpenNebula storage plugin
|
|
||||||
Vitastor storage plugin for OpenNebula.
|
|
||||||
|
|
|
@ -1,3 +1,2 @@
|
||||||
mon usr/lib/vitastor/
|
mon usr/lib/vitastor
|
||||||
mon/scripts/make-etcd usr/lib/vitastor/mon
|
mon/vitastor-mon.service /lib/systemd/system
|
||||||
mon/scripts/vitastor-mon.service /lib/systemd/system
|
|
||||||
|
|
|
@ -6,6 +6,4 @@ if [ "$1" = "configure" ]; then
|
||||||
addgroup --system --quiet vitastor
|
addgroup --system --quiet vitastor
|
||||||
adduser --system --quiet --ingroup vitastor --no-create-home --home /nonexistent vitastor
|
adduser --system --quiet --ingroup vitastor --no-create-home --home /nonexistent vitastor
|
||||||
mkdir -p /etc/vitastor
|
mkdir -p /etc/vitastor
|
||||||
mkdir -p /var/lib/vitastor
|
|
||||||
chown vitastor:vitastor /var/lib/vitastor
|
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -1,3 +0,0 @@
|
||||||
opennebula/remotes var/lib/one/
|
|
||||||
opennebula/sudoers.d etc/
|
|
||||||
opennebula/install.sh var/lib/one/remotes/datastore/vitastor/
|
|
|
@ -1,7 +0,0 @@
|
||||||
#!/bin/sh
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ "$1" = "configure" ]; then
|
|
||||||
/var/lib/one/remotes/datastore/vitastor/install.sh
|
|
||||||
fi
|
|
|
@ -1,4 +0,0 @@
|
||||||
interest /var/lib/one/remotes/datastore/downloader.sh
|
|
||||||
interest /etc/one/oned.conf
|
|
||||||
interest /etc/one/vmm_exec/vmm_execrc
|
|
||||||
interest /etc/apparmor.d/local/abstractions/libvirt-qemu
|
|
|
@ -1,6 +1,6 @@
|
||||||
usr/bin/vitastor-osd
|
usr/bin/vitastor-osd
|
||||||
usr/bin/vitastor-disk
|
usr/bin/vitastor-disk
|
||||||
usr/bin/vitastor-dump-journal
|
usr/bin/vitastor-dump-journal
|
||||||
mon/scripts/vitastor-osd@.service /lib/systemd/system
|
mon/vitastor-osd@.service /lib/systemd/system
|
||||||
mon/scripts/vitastor.target /lib/systemd/system
|
mon/vitastor.target /lib/systemd/system
|
||||||
mon/scripts/90-vitastor.rules /lib/udev/rules.d
|
mon/90-vitastor.rules /lib/udev/rules.d
|
||||||
|
|
|
@ -9,12 +9,12 @@ ARG REL=
|
||||||
|
|
||||||
WORKDIR /root
|
WORKDIR /root
|
||||||
|
|
||||||
RUN set -e -x; \
|
RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
|
||||||
if [ "$REL" = "buster" ]; then \
|
echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
|
||||||
apt-get update; \
|
echo >> /etc/apt/preferences; \
|
||||||
apt-get -y install wget; \
|
echo 'Package: *' >> /etc/apt/preferences; \
|
||||||
wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg; \
|
echo "Pin: release a=$REL-backports" >> /etc/apt/preferences; \
|
||||||
echo "deb https://vitastor.io/debian $REL main" >> /etc/apt/sources.list; \
|
echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
|
||||||
fi; \
|
fi; \
|
||||||
grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
|
grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
|
||||||
perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
|
perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
|
||||||
|
@ -22,9 +22,10 @@ RUN set -e -x; \
|
||||||
echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
|
echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
|
||||||
|
|
||||||
RUN apt-get update
|
RUN apt-get update
|
||||||
RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev libnl-3-dev libnl-genl-3-dev curl
|
RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
|
||||||
RUN apt-get -y build-dep fio
|
RUN apt-get -y build-dep fio
|
||||||
RUN apt-get --download-only source fio
|
RUN apt-get --download-only source fio
|
||||||
|
RUN apt-get update && apt-get -y install libjerasure-dev cmake libibverbs-dev libisal-dev libnl-3-dev libnl-genl-3-dev
|
||||||
|
|
||||||
ADD . /root/vitastor
|
ADD . /root/vitastor
|
||||||
RUN set -e -x; \
|
RUN set -e -x; \
|
||||||
|
@ -36,10 +37,8 @@ RUN set -e -x; \
|
||||||
mkdir -p /root/packages/vitastor-$REL; \
|
mkdir -p /root/packages/vitastor-$REL; \
|
||||||
rm -rf /root/packages/vitastor-$REL/*; \
|
rm -rf /root/packages/vitastor-$REL/*; \
|
||||||
cd /root/packages/vitastor-$REL; \
|
cd /root/packages/vitastor-$REL; \
|
||||||
FULLVER=$(head -n1 /root/vitastor/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
cp -r /root/vitastor vitastor-1.6.1; \
|
||||||
VER=${FULLVER%%-*}; \
|
cd vitastor-1.6.1; \
|
||||||
cp -r /root/vitastor vitastor-$VER; \
|
|
||||||
cd vitastor-$VER; \
|
|
||||||
ln -s /root/fio-build/fio-*/ ./fio; \
|
ln -s /root/fio-build/fio-*/ ./fio; \
|
||||||
FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||||
ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
|
ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
|
||||||
|
@ -51,14 +50,10 @@ RUN set -e -x; \
|
||||||
echo fio-headers.patch >> debian/patches/series; \
|
echo fio-headers.patch >> debian/patches/series; \
|
||||||
rm -rf a b; \
|
rm -rf a b; \
|
||||||
echo "dep:fio=$FIO" > debian/fio_version; \
|
echo "dep:fio=$FIO" > debian/fio_version; \
|
||||||
cd /root/packages/vitastor-$REL/vitastor-$VER; \
|
|
||||||
mkdir mon/node_modules; \
|
|
||||||
cd mon/node_modules; \
|
|
||||||
curl -s https://git.yourcmc.ru/vitalif/antietcd/archive/master.tar.gz | tar -zx; \
|
|
||||||
curl -s https://git.yourcmc.ru/vitalif/tinyraft/archive/master.tar.gz | tar -zx; \
|
|
||||||
cd /root/packages/vitastor-$REL; \
|
cd /root/packages/vitastor-$REL; \
|
||||||
tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_$VER.orig.tar.xz vitastor-$VER; \
|
tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.6.1.orig.tar.xz vitastor-1.6.1; \
|
||||||
cd vitastor-$VER; \
|
cd vitastor-1.6.1; \
|
||||||
DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$FULLVER""$REL" "Rebuild for $REL"; \
|
V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||||
|
DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
|
||||||
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
|
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
|
||||||
rm -rf /root/packages/vitastor-$REL/vitastor-*/
|
rm -rf /root/packages/vitastor-$REL/vitastor-*/
|
||||||
|
|
|
@ -9,7 +9,6 @@
|
||||||
These parameters apply only to Vitastor clients (QEMU, fio, NBD and so on) and
|
These parameters apply only to Vitastor clients (QEMU, fio, NBD and so on) and
|
||||||
affect their interaction with the cluster.
|
affect their interaction with the cluster.
|
||||||
|
|
||||||
- [client_iothread_count](#client_iothread_count)
|
|
||||||
- [client_retry_interval](#client_retry_interval)
|
- [client_retry_interval](#client_retry_interval)
|
||||||
- [client_eio_retry_interval](#client_eio_retry_interval)
|
- [client_eio_retry_interval](#client_eio_retry_interval)
|
||||||
- [client_retry_enospc](#client_retry_enospc)
|
- [client_retry_enospc](#client_retry_enospc)
|
||||||
|
@ -24,23 +23,6 @@ affect their interaction with the cluster.
|
||||||
- [nbd_max_part](#nbd_max_part)
|
- [nbd_max_part](#nbd_max_part)
|
||||||
- [osd_nearfull_ratio](#osd_nearfull_ratio)
|
- [osd_nearfull_ratio](#osd_nearfull_ratio)
|
||||||
|
|
||||||
## client_iothread_count
|
|
||||||
|
|
||||||
- Type: integer
|
|
||||||
- Default: 0
|
|
||||||
|
|
||||||
Number of separate threads for handling TCP network I/O at client library
|
|
||||||
side. Enabling 4 threads usually allows to increase peak performance of each
|
|
||||||
client from approx. 2-3 to 7-8 GByte/s linear read/write and from approx.
|
|
||||||
100-150 to 400 thousand iops, but at the same time it increases latency.
|
|
||||||
Latency increase depends on CPU: with CPU power saving disabled latency
|
|
||||||
only increases by ~10 us (equivalent to Q=1 iops decrease from 10500 to 9500),
|
|
||||||
with CPU power saving enabled it may be as high as 500 us (equivalent to Q=1
|
|
||||||
iops decrease from 2000 to 1000). RDMA isn't affected by this option.
|
|
||||||
|
|
||||||
It's recommended to enable client I/O threads if you don't use RDMA and want
|
|
||||||
to increase peak client performance.
|
|
||||||
|
|
||||||
## client_retry_interval
|
## client_retry_interval
|
||||||
|
|
||||||
- Type: milliseconds
|
- Type: milliseconds
|
||||||
|
|
|
@ -9,7 +9,6 @@
|
||||||
Данные параметры применяются только к клиентам Vitastor (QEMU, fio, NBD и т.п.) и
|
Данные параметры применяются только к клиентам Vitastor (QEMU, fio, NBD и т.п.) и
|
||||||
затрагивают логику их работы с кластером.
|
затрагивают логику их работы с кластером.
|
||||||
|
|
||||||
- [client_iothread_count](#client_iothread_count)
|
|
||||||
- [client_retry_interval](#client_retry_interval)
|
- [client_retry_interval](#client_retry_interval)
|
||||||
- [client_eio_retry_interval](#client_eio_retry_interval)
|
- [client_eio_retry_interval](#client_eio_retry_interval)
|
||||||
- [client_retry_enospc](#client_retry_enospc)
|
- [client_retry_enospc](#client_retry_enospc)
|
||||||
|
@ -24,24 +23,6 @@
|
||||||
- [nbd_max_part](#nbd_max_part)
|
- [nbd_max_part](#nbd_max_part)
|
||||||
- [osd_nearfull_ratio](#osd_nearfull_ratio)
|
- [osd_nearfull_ratio](#osd_nearfull_ratio)
|
||||||
|
|
||||||
## client_iothread_count
|
|
||||||
|
|
||||||
- Тип: целое число
|
|
||||||
- Значение по умолчанию: 0
|
|
||||||
|
|
||||||
Число отдельных потоков для обработки ввода-вывода через TCP сеть на стороне
|
|
||||||
клиентской библиотеки. Включение 4 потоков обычно позволяет поднять пиковую
|
|
||||||
производительность каждого клиента примерно с 2-3 до 7-8 Гбайт/с линейного
|
|
||||||
чтения/записи и примерно с 100-150 до 400 тысяч операций ввода-вывода в
|
|
||||||
секунду, но ухудшает задержку. Увеличение задержки зависит от процессора:
|
|
||||||
при отключённом энергосбережении CPU это всего ~10 микросекунд (равносильно
|
|
||||||
падению iops с Q=1 с 10500 до 9500), а при включённом это может быть
|
|
||||||
и 500 микросекунд (равносильно падению iops с Q=1 с 2000 до 1000). На работу
|
|
||||||
RDMA данная опция не влияет.
|
|
||||||
|
|
||||||
Рекомендуется включать клиентские потоки ввода-вывода, если вы не используете
|
|
||||||
RDMA и хотите повысить пиковую производительность клиентов.
|
|
||||||
|
|
||||||
## client_retry_interval
|
## client_retry_interval
|
||||||
|
|
||||||
- Тип: миллисекунды
|
- Тип: миллисекунды
|
||||||
|
|
|
@ -56,24 +56,14 @@ Can't be smaller than the OSD data device sector.
|
||||||
## immediate_commit
|
## immediate_commit
|
||||||
|
|
||||||
- Type: string
|
- Type: string
|
||||||
- Default: all
|
- Default: false
|
||||||
|
|
||||||
One of "none", "all" or "small". Global value, may be overriden [at pool level](pool.en.md#immediate_commit).
|
Another parameter which is really important for performance.
|
||||||
|
|
||||||
This parameter is also really important for performance.
|
|
||||||
|
|
||||||
TLDR: default "all" is optimal for server-grade SSDs with supercapacitor-based
|
|
||||||
power loss protection (nonvolatile write-through cache) and also for most HDDs.
|
|
||||||
"none" or "small" should be only selected if you use desktop SSDs without
|
|
||||||
capacitors or drives with slow write-back cache that can't be disabled. Check
|
|
||||||
immediate_commit of your OSDs in [ls-osd](../usage/cli.en.md#ls-osd).
|
|
||||||
|
|
||||||
Detailed explanation:
|
|
||||||
|
|
||||||
Desktop SSDs are very fast (100000+ iops) for simple random writes
|
Desktop SSDs are very fast (100000+ iops) for simple random writes
|
||||||
without cache flush. However, they are really slow (only around 1000 iops)
|
without cache flush. However, they are really slow (only around 1000 iops)
|
||||||
if you try to fsync() each write, that is, if you want to guarantee that
|
if you try to fsync() each write, that is, when you want to guarantee that
|
||||||
each change gets actually persisted to the physical media.
|
each change gets immediately persisted to the physical media.
|
||||||
|
|
||||||
Server-grade SSDs with "Advanced/Enhanced Power Loss Protection" or with
|
Server-grade SSDs with "Advanced/Enhanced Power Loss Protection" or with
|
||||||
"Supercapacitor-based Power Loss Protection", on the other hand, are equally
|
"Supercapacitor-based Power Loss Protection", on the other hand, are equally
|
||||||
|
@ -85,8 +75,8 @@ really slow when used with desktop SSDs. Vitastor, however, can also
|
||||||
efficiently utilize desktop SSDs by postponing fsync until the client calls
|
efficiently utilize desktop SSDs by postponing fsync until the client calls
|
||||||
it explicitly.
|
it explicitly.
|
||||||
|
|
||||||
This is what this parameter regulates. When it's set to "all" Vitastor
|
This is what this parameter regulates. When it's set to "all" the whole
|
||||||
cluster commits each change to disks immediately and clients just
|
Vitastor cluster commits each change to disks immediately and clients just
|
||||||
ignore fsyncs because they know for sure that they're unneeded. This reduces
|
ignore fsyncs because they know for sure that they're unneeded. This reduces
|
||||||
the amount of network roundtrips performed by clients and improves
|
the amount of network roundtrips performed by clients and improves
|
||||||
performance. So it's always better to use server grade SSDs with
|
performance. So it's always better to use server grade SSDs with
|
||||||
|
@ -109,5 +99,9 @@ Setting this parameter to "all" or "small" in OSD parameters requires enabling
|
||||||
[disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
|
[disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
|
||||||
[disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
|
[disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
|
||||||
"all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
|
"all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
|
||||||
vitastor-disk tried to do that by default, first checking/disabling drive cache.
|
|
||||||
If it can't disable drive cache, OSD get initialized with "none".
|
TLDR: For optimal performance, set immediate_commit to "all" if you only use
|
||||||
|
SSDs with supercapacitor-based power loss protection (nonvolatile
|
||||||
|
write-through cache) for both data and journals in the whole Vitastor
|
||||||
|
cluster. Set it to "small" if you only use such SSDs for journals. Leave
|
||||||
|
empty if your drives have write-back cache.
|
||||||
|
|
|
@ -57,18 +57,9 @@ amplification) и эффективность распределения нагр
|
||||||
## immediate_commit
|
## immediate_commit
|
||||||
|
|
||||||
- Тип: строка
|
- Тип: строка
|
||||||
- Значение по умолчанию: all
|
- Значение по умолчанию: false
|
||||||
|
|
||||||
Одно из значений "none", "small" или "all". Глобальное значение, может быть
|
Ещё один важный для производительности параметр.
|
||||||
переопределено [на уровне пула](pool.ru.md#immediate_commit).
|
|
||||||
|
|
||||||
Данный параметр тоже важен для производительности.
|
|
||||||
|
|
||||||
Вкратце: значение по умолчанию "all" оптимально для всех серверных SSD с
|
|
||||||
суперконденсаторами и также для большинства HDD. "none" и "small" имеет смысл
|
|
||||||
устанавливать только при использовании SSD настольного класса без
|
|
||||||
суперконденсаторов или дисков с медленным неотключаемым кэшем записи.
|
|
||||||
Проверьте настройку immediate_commit своих OSD в выводе команды [ls-osd](../usage/cli.ru.md#ls-osd).
|
|
||||||
|
|
||||||
Модели SSD для настольных компьютеров очень быстрые (100000+ операций в
|
Модели SSD для настольных компьютеров очень быстрые (100000+ операций в
|
||||||
секунду) при простой случайной записи без сбросов кэша. Однако они очень
|
секунду) при простой случайной записи без сбросов кэша. Однако они очень
|
||||||
|
@ -89,7 +80,7 @@ Power Loss Protection" - одинаково быстрые и со сбросо
|
||||||
эффективно утилизировать настольные SSD.
|
эффективно утилизировать настольные SSD.
|
||||||
|
|
||||||
Данный параметр влияет как раз на это. Когда он установлен в значение "all",
|
Данный параметр влияет как раз на это. Когда он установлен в значение "all",
|
||||||
кластер Vitastor мгновенно фиксирует каждое изменение на физические
|
весь кластер Vitastor мгновенно фиксирует каждое изменение на физические
|
||||||
носители и клиенты могут просто игнорировать запросы fsync, т.к. они точно
|
носители и клиенты могут просто игнорировать запросы fsync, т.к. они точно
|
||||||
знают, что fsync-и не нужны. Это уменьшает число необходимых обращений к OSD
|
знают, что fsync-и не нужны. Это уменьшает число необходимых обращений к OSD
|
||||||
по сети и улучшает производительность. Поэтому даже с Vitastor лучше всегда
|
по сети и улучшает производительность. Поэтому даже с Vitastor лучше всегда
|
||||||
|
@ -115,3 +106,10 @@ HDD-дисках с внутренним SSD или "медиа" кэшем - н
|
||||||
включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
|
включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
|
||||||
[disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
|
[disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
|
||||||
также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
|
также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
|
||||||
|
|
||||||
|
Итого, вкратце: для оптимальной производительности установите
|
||||||
|
immediate_commit в значение "all", если вы используете в кластере только SSD
|
||||||
|
с суперконденсаторами и для данных, и для журналов. Если вы используете
|
||||||
|
такие SSD для всех журналов, но не для данных - можете установить параметр
|
||||||
|
в "small". Если и какие-то из дисков журналов имеют волатильный кэш записи -
|
||||||
|
оставьте параметр пустым.
|
||||||
|
|
|
@ -8,14 +8,6 @@
|
||||||
|
|
||||||
These parameters only apply to Monitors.
|
These parameters only apply to Monitors.
|
||||||
|
|
||||||
- [use_antietcd](#use_antietcd)
|
|
||||||
- [enable_prometheus](#enable_prometheus)
|
|
||||||
- [mon_http_port](#mon_http_port)
|
|
||||||
- [mon_http_ip](#mon_http_ip)
|
|
||||||
- [mon_https_cert](#mon_https_cert)
|
|
||||||
- [mon_https_key](#mon_https_key)
|
|
||||||
- [mon_https_client_auth](#mon_https_client_auth)
|
|
||||||
- [mon_https_ca](#mon_https_ca)
|
|
||||||
- [etcd_mon_ttl](#etcd_mon_ttl)
|
- [etcd_mon_ttl](#etcd_mon_ttl)
|
||||||
- [etcd_mon_timeout](#etcd_mon_timeout)
|
- [etcd_mon_timeout](#etcd_mon_timeout)
|
||||||
- [etcd_mon_retries](#etcd_mon_retries)
|
- [etcd_mon_retries](#etcd_mon_retries)
|
||||||
|
@ -25,87 +17,6 @@ These parameters only apply to Monitors.
|
||||||
- [placement_levels](#placement_levels)
|
- [placement_levels](#placement_levels)
|
||||||
- [use_old_pg_combinator](#use_old_pg_combinator)
|
- [use_old_pg_combinator](#use_old_pg_combinator)
|
||||||
|
|
||||||
## use_antietcd
|
|
||||||
|
|
||||||
- Type: boolean
|
|
||||||
- Default: false
|
|
||||||
|
|
||||||
Enable experimental built-in etcd replacement (clustered key-value database):
|
|
||||||
[antietcd](https://git.yourcmc.ru/vitalif/antietcd/).
|
|
||||||
|
|
||||||
When set to true, monitor runs internal antietcd automatically if it finds
|
|
||||||
a network interface with an IP address matching one of addresses in the
|
|
||||||
`etcd_address` configuration option (in `/etc/vitastor/vitastor.conf` or in
|
|
||||||
the monitor command line). If there are multiple matching addresses, it also
|
|
||||||
checks `antietcd_port` and antietcd is started for address with matching port.
|
|
||||||
By default, antietcd accepts connection on the selected IP address, but it
|
|
||||||
can also be overridden manually in the `antietcd_ip` option.
|
|
||||||
|
|
||||||
When antietcd is started, monitor stores cluster metadata itself and exposes
|
|
||||||
a etcd-compatible REST API. On disk, these metadata are stored in
|
|
||||||
`/var/lib/vitastor/mon_2379.json.gz` (can be overridden in antietcd_data_file
|
|
||||||
or antietcd_data_dir options). All other antietcd parameters
|
|
||||||
(see [here](https://git.yourcmc.ru/vitalif/antietcd/)) except node_id,
|
|
||||||
cluster, cluster_key, persist_filter, stale_read can also be set in
|
|
||||||
Vitastor configuration with `antietcd_` prefix.
|
|
||||||
|
|
||||||
You can dump/load data to or from antietcd using Antietcd `anticli` tool:
|
|
||||||
|
|
||||||
```
|
|
||||||
npm exec anticli -e http://etcd:2379/v3 get --prefix '' --no-temp > dump.json
|
|
||||||
npm exec anticli -e http://antietcd:2379/v3 load < dump.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## enable_prometheus
|
|
||||||
|
|
||||||
- Type: boolean
|
|
||||||
- Default: true
|
|
||||||
|
|
||||||
Enable built-in Prometheus metrics exporter at mon_http_port (8060 by default).
|
|
||||||
|
|
||||||
Note that only the active (master) monitor exposes metrics, others return
|
|
||||||
HTTP 503. So you should add all monitor URLs to your Prometheus job configuration.
|
|
||||||
|
|
||||||
Grafana dashboard suitable for this exporter is here: [Vitastor-Grafana-6+.json](../../mon/scripts/Vitastor-Grafana-6+.json).
|
|
||||||
|
|
||||||
## mon_http_port
|
|
||||||
|
|
||||||
- Type: integer
|
|
||||||
- Default: 8060
|
|
||||||
|
|
||||||
HTTP port for monitors to listen on (including metrics exporter)
|
|
||||||
|
|
||||||
## mon_http_ip
|
|
||||||
|
|
||||||
- Type: string
|
|
||||||
|
|
||||||
IP address for monitors to listen on (all addresses by default)
|
|
||||||
|
|
||||||
## mon_https_cert
|
|
||||||
|
|
||||||
- Type: string
|
|
||||||
|
|
||||||
Path to PEM SSL certificate file for monitor to listen using HTTPS
|
|
||||||
|
|
||||||
## mon_https_key
|
|
||||||
|
|
||||||
- Type: string
|
|
||||||
|
|
||||||
Path to PEM SSL private key file for monitor to listen using HTTPS
|
|
||||||
|
|
||||||
## mon_https_client_auth
|
|
||||||
|
|
||||||
- Type: boolean
|
|
||||||
- Default: false
|
|
||||||
|
|
||||||
Enable HTTPS client certificate-based authorization for monitor connections
|
|
||||||
|
|
||||||
## mon_https_ca
|
|
||||||
|
|
||||||
- Type: string
|
|
||||||
|
|
||||||
Path to CA certificate for client HTTPS authorization
|
|
||||||
|
|
||||||
## etcd_mon_ttl
|
## etcd_mon_ttl
|
||||||
|
|
||||||
- Type: seconds
|
- Type: seconds
|
||||||
|
|
|
@ -8,14 +8,6 @@
|
||||||
|
|
||||||
Данные параметры используются только мониторами Vitastor.
|
Данные параметры используются только мониторами Vitastor.
|
||||||
|
|
||||||
- [use_antietcd](#use_antietcd)
|
|
||||||
- [enable_prometheus](#enable_prometheus)
|
|
||||||
- [mon_http_port](#mon_http_port)
|
|
||||||
- [mon_http_ip](#mon_http_ip)
|
|
||||||
- [mon_https_cert](#mon_https_cert)
|
|
||||||
- [mon_https_key](#mon_https_key)
|
|
||||||
- [mon_https_client_auth](#mon_https_client_auth)
|
|
||||||
- [mon_https_ca](#mon_https_ca)
|
|
||||||
- [etcd_mon_ttl](#etcd_mon_ttl)
|
- [etcd_mon_ttl](#etcd_mon_ttl)
|
||||||
- [etcd_mon_timeout](#etcd_mon_timeout)
|
- [etcd_mon_timeout](#etcd_mon_timeout)
|
||||||
- [etcd_mon_retries](#etcd_mon_retries)
|
- [etcd_mon_retries](#etcd_mon_retries)
|
||||||
|
@ -25,89 +17,6 @@
|
||||||
- [placement_levels](#placement_levels)
|
- [placement_levels](#placement_levels)
|
||||||
- [use_old_pg_combinator](#use_old_pg_combinator)
|
- [use_old_pg_combinator](#use_old_pg_combinator)
|
||||||
|
|
||||||
## use_antietcd
|
|
||||||
|
|
||||||
- Тип: булево (да/нет)
|
|
||||||
- Значение по умолчанию: false
|
|
||||||
|
|
||||||
Включить экспериментальный встроенный заменитель etcd (кластерную БД ключ-значение):
|
|
||||||
[antietcd](https://git.yourcmc.ru/vitalif/antietcd/).
|
|
||||||
|
|
||||||
Если параметр установлен в true, монитор запускает antietcd автоматически,
|
|
||||||
если обнаруживает сетевой интерфейс с одним из адресов, указанных в опции
|
|
||||||
конфигурации `etcd_address` (в `/etc/vitastor/vitastor.conf` или в опциях
|
|
||||||
командной строки монитора). Если таких адресов несколько, также проверяется
|
|
||||||
опция `antietcd_port` и antietcd запускается для адреса с соответствующим
|
|
||||||
портом. По умолчанию antietcd принимает подключения по выбранному совпадающему
|
|
||||||
IP, но его также можно определить вручную опцией `antietcd_ip`.
|
|
||||||
|
|
||||||
При запуске antietcd монитор сам хранит центральные метаданные кластера и
|
|
||||||
выставляет etcd-совместимое REST API. На диске эти метаданные хранятся в файле
|
|
||||||
`/var/lib/vitastor/mon_2379.json.gz` (можно переопределить параметрами
|
|
||||||
antietcd_data_file или antietcd_data_dir). Все остальные параметры antietcd
|
|
||||||
(смотрите [по ссылке](https://git.yourcmc.ru/vitalif/antietcd/)), за исключением
|
|
||||||
node_id, cluster, cluster_key, persist_filter, stale_read также можно задавать
|
|
||||||
в конфигурации Vitastor с префиксом `antietcd_`.
|
|
||||||
|
|
||||||
Вы можете выгружать/загружать данные в или из antietcd с помощью его инструмента
|
|
||||||
`anticli`:
|
|
||||||
|
|
||||||
```
|
|
||||||
npm exec anticli -e http://etcd:2379/v3 get --prefix '' --no-temp > dump.json
|
|
||||||
npm exec anticli -e http://antietcd:2379/v3 load < dump.json
|
|
||||||
```
|
|
||||||
|
|
||||||
## enable_prometheus
|
|
||||||
|
|
||||||
- Тип: булево (да/нет)
|
|
||||||
- Значение по умолчанию: true
|
|
||||||
|
|
||||||
Включить встроенный Prometheus-экспортер метрик на порту mon_http_port (по умолчанию 8060).
|
|
||||||
|
|
||||||
Обратите внимание, что метрики выставляет только активный (главный) монитор, остальные
|
|
||||||
возвращают статус HTTP 503, поэтому вам следует добавлять адреса всех мониторов
|
|
||||||
в задание по сбору метрик Prometheus.
|
|
||||||
|
|
||||||
Дашборд для Grafana, подходящий для этого экспортера: [Vitastor-Grafana-6+.json](../../mon/scripts/Vitastor-Grafana-6+.json).
|
|
||||||
|
|
||||||
## mon_http_port
|
|
||||||
|
|
||||||
- Тип: целое число
|
|
||||||
- Значение по умолчанию: 8060
|
|
||||||
|
|
||||||
Порт, на котором мониторы принимают HTTP-соединения (в том числе для отдачи метрик)
|
|
||||||
|
|
||||||
## mon_http_ip
|
|
||||||
|
|
||||||
- Тип: строка
|
|
||||||
|
|
||||||
IP-адрес, на котором мониторы принимают HTTP-соединения (по умолчанию все адреса)
|
|
||||||
|
|
||||||
## mon_https_cert
|
|
||||||
|
|
||||||
- Тип: строка
|
|
||||||
|
|
||||||
Путь к PEM-файлу SSL-сертификата для монитора, чтобы принимать соединения через HTTPS
|
|
||||||
|
|
||||||
## mon_https_key
|
|
||||||
|
|
||||||
- Тип: строка
|
|
||||||
|
|
||||||
Путь к PEM-файлу секретного SSL-ключа для монитора, чтобы принимать соединения через HTTPS
|
|
||||||
|
|
||||||
## mon_https_client_auth
|
|
||||||
|
|
||||||
- Тип: булево (да/нет)
|
|
||||||
- Значение по умолчанию: false
|
|
||||||
|
|
||||||
Включить в HTTPS-сервере монитора авторизацию по клиентским сертификатам
|
|
||||||
|
|
||||||
## mon_https_ca
|
|
||||||
|
|
||||||
- Тип: строка
|
|
||||||
|
|
||||||
Путь к удостоверяющему сертификату для авторизации клиентских HTTPS соединений
|
|
||||||
|
|
||||||
## etcd_mon_ttl
|
## etcd_mon_ttl
|
||||||
|
|
||||||
- Тип: секунды
|
- Тип: секунды
|
||||||
|
|
|
@ -248,7 +248,7 @@ etcd_report_interval to guarantee that keepalive actually works.
|
||||||
## etcd_ws_keepalive_interval
|
## etcd_ws_keepalive_interval
|
||||||
|
|
||||||
- Type: seconds
|
- Type: seconds
|
||||||
- Default: 5
|
- Default: 30
|
||||||
- Can be changed online: yes
|
- Can be changed online: yes
|
||||||
|
|
||||||
etcd websocket ping interval required to keep the connection alive and
|
etcd websocket ping interval required to keep the connection alive and
|
||||||
|
|
|
@ -259,7 +259,7 @@ etcd_report_interval, чтобы keepalive гарантированно рабо
|
||||||
## etcd_ws_keepalive_interval
|
## etcd_ws_keepalive_interval
|
||||||
|
|
||||||
- Тип: секунды
|
- Тип: секунды
|
||||||
- Значение по умолчанию: 5
|
- Значение по умолчанию: 30
|
||||||
- Можно менять на лету: да
|
- Можно менять на лету: да
|
||||||
|
|
||||||
Интервал проверки живости вебсокет-подключений к etcd.
|
Интервал проверки живости вебсокет-подключений к etcd.
|
||||||
|
|
|
@ -10,7 +10,6 @@ These parameters only apply to OSDs, are not fixed at the moment of OSD drive
|
||||||
initialization and can be changed - either with an OSD restart or, for some of
|
initialization and can be changed - either with an OSD restart or, for some of
|
||||||
them, even without restarting by updating configuration in etcd.
|
them, even without restarting by updating configuration in etcd.
|
||||||
|
|
||||||
- [osd_iothread_count](#osd_iothread_count)
|
|
||||||
- [etcd_report_interval](#etcd_report_interval)
|
- [etcd_report_interval](#etcd_report_interval)
|
||||||
- [etcd_stats_interval](#etcd_stats_interval)
|
- [etcd_stats_interval](#etcd_stats_interval)
|
||||||
- [run_primary](#run_primary)
|
- [run_primary](#run_primary)
|
||||||
|
@ -62,18 +61,6 @@ them, even without restarting by updating configuration in etcd.
|
||||||
- [recovery_tune_sleep_min_us](#recovery_tune_sleep_min_us)
|
- [recovery_tune_sleep_min_us](#recovery_tune_sleep_min_us)
|
||||||
- [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
|
- [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
|
||||||
|
|
||||||
## osd_iothread_count
|
|
||||||
|
|
||||||
- Type: integer
|
|
||||||
- Default: 0
|
|
||||||
|
|
||||||
TCP network I/O thread count for OSD. When non-zero, a single OSD process
|
|
||||||
may handle more TCP I/O, but at a cost of increased latency because thread
|
|
||||||
switching overhead occurs. RDMA isn't affected by this option.
|
|
||||||
|
|
||||||
Because of latency, instead of enabling OSD I/O threads it's recommended to
|
|
||||||
just create multiple OSDs per disk, or use RDMA.
|
|
||||||
|
|
||||||
## etcd_report_interval
|
## etcd_report_interval
|
||||||
|
|
||||||
- Type: seconds
|
- Type: seconds
|
||||||
|
|
|
@ -11,7 +11,6 @@
|
||||||
момент с помощью перезапуска OSD, а некоторые и без перезапуска, с помощью
|
момент с помощью перезапуска OSD, а некоторые и без перезапуска, с помощью
|
||||||
изменения конфигурации в etcd.
|
изменения конфигурации в etcd.
|
||||||
|
|
||||||
- [osd_iothread_count](#osd_iothread_count)
|
|
||||||
- [etcd_report_interval](#etcd_report_interval)
|
- [etcd_report_interval](#etcd_report_interval)
|
||||||
- [etcd_stats_interval](#etcd_stats_interval)
|
- [etcd_stats_interval](#etcd_stats_interval)
|
||||||
- [run_primary](#run_primary)
|
- [run_primary](#run_primary)
|
||||||
|
@ -63,19 +62,6 @@
|
||||||
- [recovery_tune_sleep_min_us](#recovery_tune_sleep_min_us)
|
- [recovery_tune_sleep_min_us](#recovery_tune_sleep_min_us)
|
||||||
- [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
|
- [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
|
||||||
|
|
||||||
## osd_iothread_count
|
|
||||||
|
|
||||||
- Тип: целое число
|
|
||||||
- Значение по умолчанию: 0
|
|
||||||
|
|
||||||
Число отдельных потоков для обработки ввода-вывода через TCP-сеть на
|
|
||||||
стороне OSD. Включение опции позволяет каждому отдельному OSD передавать
|
|
||||||
по сети больше данных, но ухудшает задержку из-за накладных расходов
|
|
||||||
переключения потоков. На работу RDMA опция не влияет.
|
|
||||||
|
|
||||||
Из-за задержек вместо включения потоков ввода-вывода OSD рекомендуется
|
|
||||||
просто создавать по несколько OSD на каждом диске, или использовать RDMA.
|
|
||||||
|
|
||||||
## etcd_report_interval
|
## etcd_report_interval
|
||||||
|
|
||||||
- Тип: секунды
|
- Тип: секунды
|
||||||
|
|
|
@ -1,32 +1,3 @@
|
||||||
- name: client_iothread_count
|
|
||||||
type: int
|
|
||||||
default: 0
|
|
||||||
online: false
|
|
||||||
info: |
|
|
||||||
Number of separate threads for handling TCP network I/O at client library
|
|
||||||
side. Enabling 4 threads usually allows to increase peak performance of each
|
|
||||||
client from approx. 2-3 to 7-8 GByte/s linear read/write and from approx.
|
|
||||||
100-150 to 400 thousand iops, but at the same time it increases latency.
|
|
||||||
Latency increase depends on CPU: with CPU power saving disabled latency
|
|
||||||
only increases by ~10 us (equivalent to Q=1 iops decrease from 10500 to 9500),
|
|
||||||
with CPU power saving enabled it may be as high as 500 us (equivalent to Q=1
|
|
||||||
iops decrease from 2000 to 1000). RDMA isn't affected by this option.
|
|
||||||
|
|
||||||
It's recommended to enable client I/O threads if you don't use RDMA and want
|
|
||||||
to increase peak client performance.
|
|
||||||
info_ru: |
|
|
||||||
Число отдельных потоков для обработки ввода-вывода через TCP сеть на стороне
|
|
||||||
клиентской библиотеки. Включение 4 потоков обычно позволяет поднять пиковую
|
|
||||||
производительность каждого клиента примерно с 2-3 до 7-8 Гбайт/с линейного
|
|
||||||
чтения/записи и примерно с 100-150 до 400 тысяч операций ввода-вывода в
|
|
||||||
секунду, но ухудшает задержку. Увеличение задержки зависит от процессора:
|
|
||||||
при отключённом энергосбережении CPU это всего ~10 микросекунд (равносильно
|
|
||||||
падению iops с Q=1 с 10500 до 9500), а при включённом это может быть
|
|
||||||
и 500 микросекунд (равносильно падению iops с Q=1 с 2000 до 1000). На работу
|
|
||||||
RDMA данная опция не влияет.
|
|
||||||
|
|
||||||
Рекомендуется включать клиентские потоки ввода-вывода, если вы не используете
|
|
||||||
RDMA и хотите повысить пиковую производительность клиентов.
|
|
||||||
- name: client_retry_interval
|
- name: client_retry_interval
|
||||||
type: ms
|
type: ms
|
||||||
min: 10
|
min: 10
|
||||||
|
|
|
@ -47,24 +47,14 @@
|
||||||
Не может быть меньше размера сектора дисков данных OSD.
|
Не может быть меньше размера сектора дисков данных OSD.
|
||||||
- name: immediate_commit
|
- name: immediate_commit
|
||||||
type: string
|
type: string
|
||||||
default: all
|
default: false
|
||||||
info: |
|
info: |
|
||||||
One of "none", "all" or "small". Global value, may be overriden [at pool level](pool.en.md#immediate_commit).
|
Another parameter which is really important for performance.
|
||||||
|
|
||||||
This parameter is also really important for performance.
|
|
||||||
|
|
||||||
TLDR: default "all" is optimal for server-grade SSDs with supercapacitor-based
|
|
||||||
power loss protection (nonvolatile write-through cache) and also for most HDDs.
|
|
||||||
"none" or "small" should be only selected if you use desktop SSDs without
|
|
||||||
capacitors or drives with slow write-back cache that can't be disabled. Check
|
|
||||||
immediate_commit of your OSDs in [ls-osd](../usage/cli.en.md#ls-osd).
|
|
||||||
|
|
||||||
Detailed explanation:
|
|
||||||
|
|
||||||
Desktop SSDs are very fast (100000+ iops) for simple random writes
|
Desktop SSDs are very fast (100000+ iops) for simple random writes
|
||||||
without cache flush. However, they are really slow (only around 1000 iops)
|
without cache flush. However, they are really slow (only around 1000 iops)
|
||||||
if you try to fsync() each write, that is, if you want to guarantee that
|
if you try to fsync() each write, that is, when you want to guarantee that
|
||||||
each change gets actually persisted to the physical media.
|
each change gets immediately persisted to the physical media.
|
||||||
|
|
||||||
Server-grade SSDs with "Advanced/Enhanced Power Loss Protection" or with
|
Server-grade SSDs with "Advanced/Enhanced Power Loss Protection" or with
|
||||||
"Supercapacitor-based Power Loss Protection", on the other hand, are equally
|
"Supercapacitor-based Power Loss Protection", on the other hand, are equally
|
||||||
|
@ -76,8 +66,8 @@
|
||||||
efficiently utilize desktop SSDs by postponing fsync until the client calls
|
efficiently utilize desktop SSDs by postponing fsync until the client calls
|
||||||
it explicitly.
|
it explicitly.
|
||||||
|
|
||||||
This is what this parameter regulates. When it's set to "all" Vitastor
|
This is what this parameter regulates. When it's set to "all" the whole
|
||||||
cluster commits each change to disks immediately and clients just
|
Vitastor cluster commits each change to disks immediately and clients just
|
||||||
ignore fsyncs because they know for sure that they're unneeded. This reduces
|
ignore fsyncs because they know for sure that they're unneeded. This reduces
|
||||||
the amount of network roundtrips performed by clients and improves
|
the amount of network roundtrips performed by clients and improves
|
||||||
performance. So it's always better to use server grade SSDs with
|
performance. So it's always better to use server grade SSDs with
|
||||||
|
@ -100,19 +90,14 @@
|
||||||
[disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
|
[disable_journal_fsync](layout-osd.en.yml#disable_journal_fsync) and
|
||||||
[disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
|
[disable_meta_fsync](layout-osd.en.yml#disable_meta_fsync), setting it to
|
||||||
"all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
|
"all" also requires enabling [disable_data_fsync](layout-osd.en.yml#disable_data_fsync).
|
||||||
vitastor-disk tried to do that by default, first checking/disabling drive cache.
|
|
||||||
If it can't disable drive cache, OSD get initialized with "none".
|
TLDR: For optimal performance, set immediate_commit to "all" if you only use
|
||||||
|
SSDs with supercapacitor-based power loss protection (nonvolatile
|
||||||
|
write-through cache) for both data and journals in the whole Vitastor
|
||||||
|
cluster. Set it to "small" if you only use such SSDs for journals. Leave
|
||||||
|
empty if your drives have write-back cache.
|
||||||
info_ru: |
|
info_ru: |
|
||||||
Одно из значений "none", "small" или "all". Глобальное значение, может быть
|
Ещё один важный для производительности параметр.
|
||||||
переопределено [на уровне пула](pool.ru.md#immediate_commit).
|
|
||||||
|
|
||||||
Данный параметр тоже важен для производительности.
|
|
||||||
|
|
||||||
Вкратце: значение по умолчанию "all" оптимально для всех серверных SSD с
|
|
||||||
суперконденсаторами и также для большинства HDD. "none" и "small" имеет смысл
|
|
||||||
устанавливать только при использовании SSD настольного класса без
|
|
||||||
суперконденсаторов или дисков с медленным неотключаемым кэшем записи.
|
|
||||||
Проверьте настройку immediate_commit своих OSD в выводе команды [ls-osd](../usage/cli.ru.md#ls-osd).
|
|
||||||
|
|
||||||
Модели SSD для настольных компьютеров очень быстрые (100000+ операций в
|
Модели SSD для настольных компьютеров очень быстрые (100000+ операций в
|
||||||
секунду) при простой случайной записи без сбросов кэша. Однако они очень
|
секунду) при простой случайной записи без сбросов кэша. Однако они очень
|
||||||
|
@ -133,7 +118,7 @@
|
||||||
эффективно утилизировать настольные SSD.
|
эффективно утилизировать настольные SSD.
|
||||||
|
|
||||||
Данный параметр влияет как раз на это. Когда он установлен в значение "all",
|
Данный параметр влияет как раз на это. Когда он установлен в значение "all",
|
||||||
кластер Vitastor мгновенно фиксирует каждое изменение на физические
|
весь кластер Vitastor мгновенно фиксирует каждое изменение на физические
|
||||||
носители и клиенты могут просто игнорировать запросы fsync, т.к. они точно
|
носители и клиенты могут просто игнорировать запросы fsync, т.к. они точно
|
||||||
знают, что fsync-и не нужны. Это уменьшает число необходимых обращений к OSD
|
знают, что fsync-и не нужны. Это уменьшает число необходимых обращений к OSD
|
||||||
по сети и улучшает производительность. Поэтому даже с Vitastor лучше всегда
|
по сети и улучшает производительность. Поэтому даже с Vitastor лучше всегда
|
||||||
|
@ -159,3 +144,10 @@
|
||||||
включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
|
включения [disable_journal_fsync](layout-osd.ru.yml#disable_journal_fsync) и
|
||||||
[disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
|
[disable_meta_fsync](layout-osd.ru.yml#disable_meta_fsync), значение "all"
|
||||||
также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
|
также требует включения [disable_data_fsync](layout-osd.ru.yml#disable_data_fsync).
|
||||||
|
|
||||||
|
Итого, вкратце: для оптимальной производительности установите
|
||||||
|
immediate_commit в значение "all", если вы используете в кластере только SSD
|
||||||
|
с суперконденсаторами и для данных, и для журналов. Если вы используете
|
||||||
|
такие SSD для всех журналов, но не для данных - можете установить параметр
|
||||||
|
в "small". Если и какие-то из дисков журналов имеют волатильный кэш записи -
|
||||||
|
оставьте параметр пустым.
|
||||||
|
|
|
@ -1,103 +1,3 @@
|
||||||
- name: use_antietcd
|
|
||||||
type: bool
|
|
||||||
default: false
|
|
||||||
info: |
|
|
||||||
Enable experimental built-in etcd replacement (clustered key-value database):
|
|
||||||
[antietcd](https://git.yourcmc.ru/vitalif/antietcd/).
|
|
||||||
|
|
||||||
When set to true, monitor runs internal antietcd automatically if it finds
|
|
||||||
a network interface with an IP address matching one of addresses in the
|
|
||||||
`etcd_address` configuration option (in `/etc/vitastor/vitastor.conf` or in
|
|
||||||
the monitor command line). If there are multiple matching addresses, it also
|
|
||||||
checks `antietcd_port` and antietcd is started for address with matching port.
|
|
||||||
By default, antietcd accepts connection on the selected IP address, but it
|
|
||||||
can also be overridden manually in the `antietcd_ip` option.
|
|
||||||
|
|
||||||
When antietcd is started, monitor stores cluster metadata itself and exposes
|
|
||||||
a etcd-compatible REST API. On disk, these metadata are stored in
|
|
||||||
`/var/lib/vitastor/mon_2379.json.gz` (can be overridden in antietcd_data_file
|
|
||||||
or antietcd_data_dir options). All other antietcd parameters
|
|
||||||
(see [here](https://git.yourcmc.ru/vitalif/antietcd/)) except node_id,
|
|
||||||
cluster, cluster_key, persist_filter, stale_read can also be set in
|
|
||||||
Vitastor configuration with `antietcd_` prefix.
|
|
||||||
|
|
||||||
You can dump/load data to or from antietcd using Antietcd `anticli` tool:
|
|
||||||
|
|
||||||
```
|
|
||||||
npm exec anticli -e http://etcd:2379/v3 get --prefix '' --no-temp > dump.json
|
|
||||||
npm exec anticli -e http://antietcd:2379/v3 load < dump.json
|
|
||||||
```
|
|
||||||
info_ru: |
|
|
||||||
Включить экспериментальный встроенный заменитель etcd (кластерную БД ключ-значение):
|
|
||||||
[antietcd](https://git.yourcmc.ru/vitalif/antietcd/).
|
|
||||||
|
|
||||||
Если параметр установлен в true, монитор запускает antietcd автоматически,
|
|
||||||
если обнаруживает сетевой интерфейс с одним из адресов, указанных в опции
|
|
||||||
конфигурации `etcd_address` (в `/etc/vitastor/vitastor.conf` или в опциях
|
|
||||||
командной строки монитора). Если таких адресов несколько, также проверяется
|
|
||||||
опция `antietcd_port` и antietcd запускается для адреса с соответствующим
|
|
||||||
портом. По умолчанию antietcd принимает подключения по выбранному совпадающему
|
|
||||||
IP, но его также можно определить вручную опцией `antietcd_ip`.
|
|
||||||
|
|
||||||
При запуске antietcd монитор сам хранит центральные метаданные кластера и
|
|
||||||
выставляет etcd-совместимое REST API. На диске эти метаданные хранятся в файле
|
|
||||||
`/var/lib/vitastor/mon_2379.json.gz` (можно переопределить параметрами
|
|
||||||
antietcd_data_file или antietcd_data_dir). Все остальные параметры antietcd
|
|
||||||
(смотрите [по ссылке](https://git.yourcmc.ru/vitalif/antietcd/)), за исключением
|
|
||||||
node_id, cluster, cluster_key, persist_filter, stale_read также можно задавать
|
|
||||||
в конфигурации Vitastor с префиксом `antietcd_`.
|
|
||||||
|
|
||||||
Вы можете выгружать/загружать данные в или из antietcd с помощью его инструмента
|
|
||||||
`anticli`:
|
|
||||||
|
|
||||||
```
|
|
||||||
npm exec anticli -e http://etcd:2379/v3 get --prefix '' --no-temp > dump.json
|
|
||||||
npm exec anticli -e http://antietcd:2379/v3 load < dump.json
|
|
||||||
```
|
|
||||||
- name: enable_prometheus
|
|
||||||
type: bool
|
|
||||||
default: true
|
|
||||||
info: |
|
|
||||||
Enable built-in Prometheus metrics exporter at mon_http_port (8060 by default).
|
|
||||||
|
|
||||||
Note that only the active (master) monitor exposes metrics, others return
|
|
||||||
HTTP 503. So you should add all monitor URLs to your Prometheus job configuration.
|
|
||||||
|
|
||||||
Grafana dashboard suitable for this exporter is here: [Vitastor-Grafana-6+.json](../../mon/scripts/Vitastor-Grafana-6+.json).
|
|
||||||
info_ru: |
|
|
||||||
Включить встроенный Prometheus-экспортер метрик на порту mon_http_port (по умолчанию 8060).
|
|
||||||
|
|
||||||
Обратите внимание, что метрики выставляет только активный (главный) монитор, остальные
|
|
||||||
возвращают статус HTTP 503, поэтому вам следует добавлять адреса всех мониторов
|
|
||||||
в задание по сбору метрик Prometheus.
|
|
||||||
|
|
||||||
Дашборд для Grafana, подходящий для этого экспортера: [Vitastor-Grafana-6+.json](../../mon/scripts/Vitastor-Grafana-6+.json).
|
|
||||||
- name: mon_http_port
|
|
||||||
type: int
|
|
||||||
default: 8060
|
|
||||||
info: HTTP port for monitors to listen on (including metrics exporter)
|
|
||||||
info_ru: Порт, на котором мониторы принимают HTTP-соединения (в том числе для отдачи метрик)
|
|
||||||
- name: mon_http_ip
|
|
||||||
type: string
|
|
||||||
info: IP address for monitors to listen on (all addresses by default)
|
|
||||||
info_ru: IP-адрес, на котором мониторы принимают HTTP-соединения (по умолчанию все адреса)
|
|
||||||
- name: mon_https_cert
|
|
||||||
type: string
|
|
||||||
info: Path to PEM SSL certificate file for monitor to listen using HTTPS
|
|
||||||
info_ru: Путь к PEM-файлу SSL-сертификата для монитора, чтобы принимать соединения через HTTPS
|
|
||||||
- name: mon_https_key
|
|
||||||
type: string
|
|
||||||
info: Path to PEM SSL private key file for monitor to listen using HTTPS
|
|
||||||
info_ru: Путь к PEM-файлу секретного SSL-ключа для монитора, чтобы принимать соединения через HTTPS
|
|
||||||
- name: mon_https_client_auth
|
|
||||||
type: bool
|
|
||||||
default: false
|
|
||||||
info: Enable HTTPS client certificate-based authorization for monitor connections
|
|
||||||
info_ru: Включить в HTTPS-сервере монитора авторизацию по клиентским сертификатам
|
|
||||||
- name: mon_https_ca
|
|
||||||
type: string
|
|
||||||
info: Path to CA certificate for client HTTPS authorization
|
|
||||||
info_ru: Путь к удостоверяющему сертификату для авторизации клиентских HTTPS соединений
|
|
||||||
- name: etcd_mon_ttl
|
- name: etcd_mon_ttl
|
||||||
type: sec
|
type: sec
|
||||||
min: 5
|
min: 5
|
||||||
|
|
|
@ -282,7 +282,7 @@
|
||||||
etcd_report_interval, чтобы keepalive гарантированно работал.
|
etcd_report_interval, чтобы keepalive гарантированно работал.
|
||||||
- name: etcd_ws_keepalive_interval
|
- name: etcd_ws_keepalive_interval
|
||||||
type: sec
|
type: sec
|
||||||
default: 5
|
default: 30
|
||||||
online: true
|
online: true
|
||||||
info: |
|
info: |
|
||||||
etcd websocket ping interval required to keep the connection alive and
|
etcd websocket ping interval required to keep the connection alive and
|
||||||
|
|
|
@ -1,21 +1,3 @@
|
||||||
- name: osd_iothread_count
|
|
||||||
type: int
|
|
||||||
default: 0
|
|
||||||
info: |
|
|
||||||
TCP network I/O thread count for OSD. When non-zero, a single OSD process
|
|
||||||
may handle more TCP I/O, but at a cost of increased latency because thread
|
|
||||||
switching overhead occurs. RDMA isn't affected by this option.
|
|
||||||
|
|
||||||
Because of latency, instead of enabling OSD I/O threads it's recommended to
|
|
||||||
just create multiple OSDs per disk, or use RDMA.
|
|
||||||
info_ru: |
|
|
||||||
Число отдельных потоков для обработки ввода-вывода через TCP-сеть на
|
|
||||||
стороне OSD. Включение опции позволяет каждому отдельному OSD передавать
|
|
||||||
по сети больше данных, но ухудшает задержку из-за накладных расходов
|
|
||||||
переключения потоков. На работу RDMA опция не влияет.
|
|
||||||
|
|
||||||
Из-за задержек вместо включения потоков ввода-вывода OSD рекомендуется
|
|
||||||
просто создавать по несколько OSD на каждом диске, или использовать RDMA.
|
|
||||||
- name: etcd_report_interval
|
- name: etcd_report_interval
|
||||||
type: sec
|
type: sec
|
||||||
default: 5
|
default: 5
|
||||||
|
|
|
@ -1,184 +0,0 @@
|
||||||
[Documentation](../../README.md#documentation) → Installation → OpenNebula
|
|
||||||
|
|
||||||
-----
|
|
||||||
|
|
||||||
[Читать на русском](opennebula.ru.md)
|
|
||||||
|
|
||||||
## Automatic Installation
|
|
||||||
|
|
||||||
OpenNebula plugin is packaged as `vitastor-opennebula` Debian and RPM package since Vitastor 1.9.0. So:
|
|
||||||
|
|
||||||
- Run `apt-get install vitastor-opennebula` or `yum install vitastor-opennebula` after installing OpenNebula on all nodes
|
|
||||||
- Check that it prints "OK, Vitastor OpenNebula patches successfully applied" or "OK, Vitastor OpenNebula patches are already applied"
|
|
||||||
- If it does not, refer to [Manual Installation](#manual-installation) and apply configuration file changes manually
|
|
||||||
- Make sure that Vitastor patched versions of QEMU and libvirt are installed
|
|
||||||
(`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` should show "vitastor" in version names)
|
|
||||||
- [Block VM access to Vitastor cluster](#block-vm-access-to-vitastor-cluster)
|
|
||||||
|
|
||||||
## Manual Installation
|
|
||||||
|
|
||||||
Install OpenNebula. Then, on each node:
|
|
||||||
|
|
||||||
- Copy [opennebula/remotes](../../opennebula/remotes) into `/var/lib/one` recursively: `cp -r opennebula/remotes /var/lib/one/`
|
|
||||||
- Copy [opennebula/sudoers.d](../../opennebula/sudoers.d) to `/etc`: `cp -r opennebula/sudoers.d /etc/`
|
|
||||||
- Apply [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) to `/var/lib/one/remotes/datastore/downloader.sh`:
|
|
||||||
`patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - or read the patch and apply the same change manually
|
|
||||||
- Add `kvm-vitastor` to `LIVE_DISK_SNAPSHOTS` in `/etc/one/vmm_exec/vmm_execrc`
|
|
||||||
- If on Debian or Ubuntu (and AppArmor is used), add Vitastor config file path(s) to `/etc/apparmor.d/local/abstractions/libvirt-qemu`: for example,
|
|
||||||
`echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
|
|
||||||
- Apply changes to `/etc/one/oned.conf`
|
|
||||||
|
|
||||||
### oned.conf changes
|
|
||||||
|
|
||||||
1. Add deploy script override in kvm VM_MAD: add `-l deploy.vitastor` to ARGUMENTS.
|
|
||||||
|
|
||||||
```diff
|
|
||||||
VM_MAD = [
|
|
||||||
NAME = "kvm",
|
|
||||||
SUNSTONE_NAME = "KVM",
|
|
||||||
EXECUTABLE = "one_vmm_exec",
|
|
||||||
- ARGUMENTS = "-t 15 -r 0 kvm -p",
|
|
||||||
+ ARGUMENTS = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
|
|
||||||
DEFAULT = "vmm_exec/vmm_exec_kvm.conf",
|
|
||||||
TYPE = "kvm",
|
|
||||||
KEEP_SNAPSHOTS = "yes",
|
|
||||||
LIVE_RESIZE = "yes",
|
|
||||||
SUPPORT_SHAREABLE = "yes",
|
|
||||||
IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
|
|
||||||
resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
|
|
||||||
disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
|
|
||||||
resize, updateconf, update"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
Optional: if you also want to save VM RAM checkpoints to Vitastor, use
|
|
||||||
`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
|
|
||||||
instead of just `-l deploy=deploy.vitastor`.
|
|
||||||
|
|
||||||
2. Add `vitastor` to TM_MAD.ARGUMENTS and DATASTORE_MAD.ARGUMENTS:
|
|
||||||
|
|
||||||
```diff
|
|
||||||
TM_MAD = [
|
|
||||||
EXECUTABLE = "one_tm",
|
|
||||||
- ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
|
|
||||||
+ ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
|
|
||||||
]
|
|
||||||
|
|
||||||
DATASTORE_MAD = [
|
|
||||||
EXECUTABLE = "one_datastore",
|
|
||||||
- ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
|
|
||||||
+ ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Add INHERIT_DATASTORE_ATTR for two Vitastor attributes:
|
|
||||||
|
|
||||||
```
|
|
||||||
INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
|
|
||||||
INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
|
|
||||||
```
|
|
||||||
|
|
||||||
4. Add TM_MAD_CONF and DS_MAD_CONF for Vitastor:
|
|
||||||
|
|
||||||
```
|
|
||||||
TM_MAD_CONF = [
|
|
||||||
NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
|
|
||||||
DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
|
|
||||||
TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
|
|
||||||
DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
|
|
||||||
CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
|
|
||||||
]
|
|
||||||
|
|
||||||
DS_MAD_CONF = [
|
|
||||||
NAME = "vitastor",
|
|
||||||
REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
|
|
||||||
PERSISTENT_ONLY = "NO",
|
|
||||||
MARKETPLACE_ACTIONS = "export"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Create Datastores
|
|
||||||
|
|
||||||
Example Image and System Datastore definitions:
|
|
||||||
[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) and
|
|
||||||
[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
|
|
||||||
|
|
||||||
Change parameters to your will:
|
|
||||||
|
|
||||||
- POOL_NAME is Vitastor pool name to store images.
|
|
||||||
- IMAGE_PREFIX is a string prepended to all Vitastor image names.
|
|
||||||
- BRIDGE_LIST is a list of hosts with access to Vitastor cluster, mostly used for image (not system) datastore operations.
|
|
||||||
- VITASTOR_CONF is the path to cluster configuration. Note that it should be also added to `/etc/apparmor.d/local/abstractions/libvirt-qemu` if you use AppArmor.
|
|
||||||
- STAGING_DIR is a temporary directory used when importing external images. Should have free space sufficient for downloading external images.
|
|
||||||
|
|
||||||
Then create datastores using `onedatastore create vitastor-imageds.conf` and `onedatastore create vitastor-systemds.conf` (or use UI).
|
|
||||||
|
|
||||||
## Block VM access to Vitastor cluster
|
|
||||||
|
|
||||||
Vitastor doesn't support any authentication yet, so you MUST block VM guest access to the Vitastor cluster at the network level.
|
|
||||||
|
|
||||||
If you use VLAN networking for VMs - make sure you use different VLANs for VMs and hypervisor/storage network and
|
|
||||||
block access between them using your firewall/switch configuration.
|
|
||||||
|
|
||||||
If you use something more stupid like bridged networking, you probably have to use manual firewall/iptables setup
|
|
||||||
to only allow access to Vitastor from hypervisor IPs.
|
|
||||||
|
|
||||||
Also you need to switch network to "Bridged & Security Groups" and enable IP spoofing filters in OpenNebula.
|
|
||||||
Problem is that OpenNebula's IP spoofing filter doesn't affect local interfaces of the hypervisor i.e. when
|
|
||||||
it's enabled a VM can't talk to other VMs or to the outer world using a spoofed IP, but it CAN talk to the
|
|
||||||
hypervisor if it takes an IP from its subnet. To fix that you also need some more iptables.
|
|
||||||
|
|
||||||
So the complete "stupid" bridged network filter setup could look like the following
|
|
||||||
(here `10.0.3.0/24` is the VM subnet and `10.0.2.0/24` is the hypervisor subnet):
|
|
||||||
|
|
||||||
```
|
|
||||||
# Allow incoming traffic from physical device
|
|
||||||
iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
|
|
||||||
# Do not allow incoming traffic from VMs, but not from VM subnet
|
|
||||||
iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
|
|
||||||
# Drop traffic from VMs to hypervisor/storage subnet
|
|
||||||
iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
|
|
||||||
```
|
|
||||||
|
|
||||||
## Testing
|
|
||||||
|
|
||||||
The OpenNebula plugin includes quite a bit of bash scripts, so here's their description to get an idea about what they actually do.
|
|
||||||
|
|
||||||
| Script | Action | How to Test |
|
|
||||||
| ----------------------- | ----------------------------------------- | ------------------------------------------------------------------------------------ |
|
|
||||||
| vmm/kvm/deploy.vitastor | Start a VM | Create and start a VM with Vitastor disk(s): persistent / non-persistent / volatile. |
|
|
||||||
| vmm/kvm/save.vitastor | Save VM memory checkpoint | Stop a VM using "Stop" command. |
|
|
||||||
| vmm/kvm/restore.vitastor| Restore VM memory checkpoint | Start a VM back after stopping it. |
|
|
||||||
| datastore/clone | Copy an image as persistent | Create a VM template and instantiate it as persistent. |
|
|
||||||
| datastore/cp | Import an external image | Import a VM template with images from Marketplace. |
|
|
||||||
| datastore/export | Export an image as URL | Probably: export a VM template with images to Marketplace. |
|
|
||||||
| datastore/mkfs | Create an image with FS | Storage → Images → Create → Type: Datablock, Location: Empty disk image, Filesystem: Not empty. |
|
|
||||||
| datastore/monitor | Monitor used space in image datastore | Check reported used/free space in image datastore list. |
|
|
||||||
| datastore/rm | Remove a persistent image | Storage → Images → Select an image → Delete. |
|
|
||||||
| datastore/snap_delete | Delete a snapshot of a persistent image | Storage → Images → Select an image → Select a snapshot → Delete; <br> To create an image with snapshot: attach a persistent image to a VM; create a snapshot; detach the image. |
|
|
||||||
| datastore/snap_flatten | Revert an image to snapshot and delete other snapshots | Storage → Images → Select an image → Select a snapshot → Flatten. |
|
|
||||||
| datastore/snap_revert | Revert an image to snapshot | Storage → Images → Select an image → Select a snapshot → Revert. |
|
|
||||||
| datastore/stat | Get virtual size of an image in MB | No idea. Seems to be unused both in Vitastor and Ceph datastores. |
|
|
||||||
| tm/clone | Clone a non-persistent image to a VM disk | Attach a non-persistent image to a VM. |
|
|
||||||
| tm/context | Generate a contextualisation VM disk | Create a VM with enabled contextualisation (default). Common host FS-based version is used in Vitastor and Ceph datastores. |
|
|
||||||
| tm/cpds | Copy a VM disk / its snapshot to an image | Select a VM → Select a disk → Optionally select a snapshot → Save as. |
|
|
||||||
| tm/delete | Delete a cloned or volatile VM disk | Detach a volatile disk or a non-persistent image from a VM. |
|
|
||||||
| tm/failmigrate | Handle live migration failure | No action. Script is empty in Vitastor and Ceph. In other datastores, should roll back actions done by tm/premigrate. |
|
|
||||||
| tm/ln | Attach a persistent image to a VM | No action. Script is empty in Vitastor and Ceph. |
|
|
||||||
| tm/mkimage | Create a volatile disk, maybe with FS | Attach a volatile disk to a VM, with or without file system. |
|
|
||||||
| tm/mkswap | Create a volatile swap disk | Attach a volatile disk to a VM, formatted as swap. |
|
|
||||||
| tm/monitor | Monitor used space in system datastore | Check reported used/free space in system datastore list. |
|
|
||||||
| tm/mv | Move a migrated VM disk between hosts | Migrate a VM between hosts. In Vitastor and Ceph datastores, doesn't do any storage action. |
|
|
||||||
| tm/mvds | Detach a persistent image from a VM | No action. The opposite of tm/ln. Script is empty in Vitastor and Ceph. In other datastores, script may copy the image from VM host back to the datastore. |
|
|
||||||
| tm/postbackup | Executed after backup | Seems that the script just removes temporary files after backup. Perform a VM backup and check that temporary files are cleaned up. |
|
|
||||||
| tm/postbackup_live | Executed after backup of a running VM | Same as tm/postbackup, but for a running VM. |
|
|
||||||
| tm/postmigrate | Executed after VM live migration | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
|
|
||||||
| tm/prebackup | Actual backup script: backup VM disks | Set up "rsync" backup datastore → Backup a VM to it. |
|
|
||||||
| tm/prebackup_live | Backup VM disks of a running VM | Same as tm/prebackup, but also does fsfreeze/thaw. So perform a live backup, restore it and check that disks are consistent. |
|
|
||||||
| tm/premigrate | Executed before live migration | No action. Only executed for system datastore, so the script tries to call other TMs for other disks. Except that, the script does nothing in Vitastor and Ceph datastores. |
|
|
||||||
| tm/resize | Resize a VM disk | Select a VM → Select a non-persistent disk → Resize. |
|
|
||||||
| tm/restore | Restore VM disks from backup | Set up "rsync" backup datastore → Backup a VM to it → Restore it back. |
|
|
||||||
| tm/snap_create | Create a VM disk snapshot | Select a VM → Select a disk → Create snapshot. |
|
|
||||||
| tm/snap_create_live | Create a VM disk snapshot for a live VM | Select a running VM → Select a disk → Create snapshot. |
|
|
||||||
| tm/snap_delete | Delete a VM disk snapshot | Select a VM → Select a disk → Select a snapshot → Delete. |
|
|
||||||
| tm/snap_revert | Revert a VM disk to a snapshot | Select a VM → Select a disk → Select a snapshot → Revert. |
|
|
|
@ -1,187 +0,0 @@
|
||||||
[Документация](../../README-ru.md#документация) → Установка → OpenNebula
|
|
||||||
|
|
||||||
-----
|
|
||||||
|
|
||||||
[Read in English](opennebula.en.md)
|
|
||||||
|
|
||||||
## Автоматическая установка
|
|
||||||
|
|
||||||
Плагин OpenNebula Vitastor распространяется как Debian и RPM пакет `vitastor-opennebula`, начиная с версии Vitastor 1.9.0. Так что:
|
|
||||||
|
|
||||||
- Запустите `apt-get install vitastor-opennebula` или `yum install vitastor-opennebula` после установки OpenNebula на всех серверах
|
|
||||||
- Проверьте, что он выводит "OK, Vitastor OpenNebula patches successfully applied" или "OK, Vitastor OpenNebula patches are already applied" в процессе установки
|
|
||||||
- Если сообщение не выведено, пройдите по шагам инструкцию [Ручная установка](#ручная-установка) и примените правки файлов конфигурации вручную
|
|
||||||
- Удостоверьтесь, что установлены версии QEMU и libvirt с изменениями Vitastor
|
|
||||||
(`dpkg -l qemu-system-x86`, `dpkg -l | grep libvirt`, `rpm -qa | grep qemu`, `rpm -qa | grep qemu`, `rpm -qa | grep libvirt-libs` должны показывать "vitastor" в номере версии)
|
|
||||||
- [Заблокируйте доступ виртуальных машин в Vitastor](#блокировка-доступа-вм-в-vitastor)
|
|
||||||
|
|
||||||
## Ручная установка
|
|
||||||
|
|
||||||
Сначала установите саму OpenNebula. После этого, на каждом сервере:
|
|
||||||
|
|
||||||
- Скопируйте директорию [opennebula/remotes](../../opennebula/remotes) в `/var/lib/one`: `cp -r opennebula/remotes /var/lib/one/`
|
|
||||||
- Скопируйте директорию [opennebula/sudoers.d](../../opennebula/sudoers.d) в `/etc`: `cp -r opennebula/sudoers.d /etc/`
|
|
||||||
- Примените патч [downloader-vitastor.sh.diff](../../opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff) к `/var/lib/one/remotes/datastore/downloader.sh`:
|
|
||||||
`patch /var/lib/one/remotes/datastore/downloader.sh < opennebula/remotes/datastore/vitastor/downloader-vitastor.sh.diff` - либо прочитайте патч и примените изменение вручную
|
|
||||||
- Добавьте `kvm-vitastor` в список `LIVE_DISK_SNAPSHOTS` в файле `/etc/one/vmm_exec/vmm_execrc`
|
|
||||||
- Если вы используете Debian или Ubuntu (и AppArmor), добавьте пути к файлу(ам) конфигурации Vitastor в файл `/etc/apparmor.d/local/abstractions/libvirt-qemu`: например,
|
|
||||||
`echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu`
|
|
||||||
- Примените изменения `/etc/one/oned.conf`
|
|
||||||
|
|
||||||
### Изменения oned.conf
|
|
||||||
|
|
||||||
1. Добавьте переопределение скрипта deploy в VM_MAD kvm, добавив `-l deploy.vitastor` в `ARGUMENTS`:
|
|
||||||
|
|
||||||
```diff
|
|
||||||
VM_MAD = [
|
|
||||||
NAME = "kvm",
|
|
||||||
SUNSTONE_NAME = "KVM",
|
|
||||||
EXECUTABLE = "one_vmm_exec",
|
|
||||||
- ARGUMENTS = "-t 15 -r 0 kvm -p",
|
|
||||||
+ ARGUMENTS = "-t 15 -r 0 kvm -p -l deploy=deploy.vitastor",
|
|
||||||
DEFAULT = "vmm_exec/vmm_exec_kvm.conf",
|
|
||||||
TYPE = "kvm",
|
|
||||||
KEEP_SNAPSHOTS = "yes",
|
|
||||||
LIVE_RESIZE = "yes",
|
|
||||||
SUPPORT_SHAREABLE = "yes",
|
|
||||||
IMPORTED_VMS_ACTIONS = "terminate, terminate-hard, hold, release, suspend,
|
|
||||||
resume, delete, reboot, reboot-hard, resched, unresched, disk-attach,
|
|
||||||
disk-detach, nic-attach, nic-detach, snapshot-create, snapshot-delete,
|
|
||||||
resize, updateconf, update"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
Опционально: если вы хотите также сохранять снимки памяти ВМ в Vitastor, добавьте
|
|
||||||
`-l deploy=deploy.vitastor,save=save.vitastor,restore=restore.vitastor`
|
|
||||||
вместо просто `-l deploy=deploy.vitastor`.
|
|
||||||
|
|
||||||
2. Добавьте `vitastor` в значения TM_MAD.ARGUMENTS и DATASTORE_MAD.ARGUMENTS:
|
|
||||||
|
|
||||||
```diff
|
|
||||||
TM_MAD = [
|
|
||||||
EXECUTABLE = "one_tm",
|
|
||||||
- ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,dev,vcenter,iscsi_libvirt"
|
|
||||||
+ ARGUMENTS = "-t 15 -d dummy,lvm,shared,fs_lvm,fs_lvm_ssh,qcow2,ssh,ceph,vitastor,dev,vcenter,iscsi_libvirt"
|
|
||||||
]
|
|
||||||
|
|
||||||
DATASTORE_MAD = [
|
|
||||||
EXECUTABLE = "one_datastore",
|
|
||||||
- ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
|
|
||||||
+ ARGUMENTS = "-t 15 -d dummy,fs,lvm,ceph,vitastor,dev,iscsi_libvirt,vcenter,restic,rsync -s shared,ssh,ceph,vitastor,fs_lvm,fs_lvm_ssh,qcow2,vcenter"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
3. Добавьте строчки с INHERIT_DATASTORE_ATTR для двух атрибутов Vitastor-хранилищ:
|
|
||||||
|
|
||||||
```
|
|
||||||
INHERIT_DATASTORE_ATTR = "VITASTOR_CONF"
|
|
||||||
INHERIT_DATASTORE_ATTR = "IMAGE_PREFIX"
|
|
||||||
```
|
|
||||||
|
|
||||||
4. Добавьте TM_MAD_CONF и DS_MAD_CONF для Vitastor:
|
|
||||||
|
|
||||||
```
|
|
||||||
TM_MAD_CONF = [
|
|
||||||
NAME = "vitastor", LN_TARGET = "NONE", CLONE_TARGET = "SELF", SHARED = "YES",
|
|
||||||
DS_MIGRATE = "NO", DRIVER = "raw", ALLOW_ORPHANS="format",
|
|
||||||
TM_MAD_SYSTEM = "ssh,shared", LN_TARGET_SSH = "SYSTEM", CLONE_TARGET_SSH = "SYSTEM",
|
|
||||||
DISK_TYPE_SSH = "FILE", LN_TARGET_SHARED = "NONE",
|
|
||||||
CLONE_TARGET_SHARED = "SELF", DISK_TYPE_SHARED = "FILE"
|
|
||||||
]
|
|
||||||
|
|
||||||
DS_MAD_CONF = [
|
|
||||||
NAME = "vitastor",
|
|
||||||
REQUIRED_ATTRS = "DISK_TYPE,BRIDGE_LIST",
|
|
||||||
PERSISTENT_ONLY = "NO",
|
|
||||||
MARKETPLACE_ACTIONS = "export"
|
|
||||||
]
|
|
||||||
```
|
|
||||||
|
|
||||||
## Создайте хранилища
|
|
||||||
|
|
||||||
Примеры настроек хранилищ образов (image) и дисков ВМ (system):
|
|
||||||
[opennebula/vitastor-imageds.conf](../../opennebula/vitastor-imageds.conf) и
|
|
||||||
[opennebula/vitastor-systemds.conf](../../opennebula/vitastor-systemds.conf).
|
|
||||||
|
|
||||||
Скопируйте настройки и поменяйте следующие параметры так, как вам необходимо:
|
|
||||||
|
|
||||||
- POOL_NAME - имя пула Vitastor для сохранения образов дисков.
|
|
||||||
- IMAGE_PREFIX - строка, добавляемая в начало имён образов дисков.
|
|
||||||
- BRIDGE_LIST - список серверов с доступом к кластеру Vitastor, используемых для операций с хранилищем образов (image, не system).
|
|
||||||
- VITASTOR_CONF - путь к конфигурации Vitastor. Имейте в виду, что этот путь также надо добавить в `/etc/apparmor.d/local/abstractions/libvirt-qemu`, если вы используете AppArmor.
|
|
||||||
- STAGING_DIR - путь к временному каталогу, используемому при импорте внешних образов. Должен иметь достаточно свободного места, чтобы вмещать скачанные образы.
|
|
||||||
|
|
||||||
После этого создайте хранилища с помощью команд `onedatastore create vitastor-imageds.conf` и `onedatastore create vitastor-systemds.conf` (либо через UI).
|
|
||||||
|
|
||||||
## Блокировка доступа ВМ в Vitastor
|
|
||||||
|
|
||||||
Vitastor пока не поддерживает никакую аутентификацию, так что вы ДОЛЖНЫ заблокировать доступ гостевых ВМ
|
|
||||||
в кластер Vitastor на сетевом уровне.
|
|
||||||
|
|
||||||
Если вы используете VLAN-сети для ВМ - удостоверьтесь, что ВМ и гипервизор/сеть хранения помещены в разные
|
|
||||||
изолированные друг от друга VLAN-ы.
|
|
||||||
|
|
||||||
Если вы используете что-то более примитивное, например, мосты (bridge), вам, скорее всего, придётся вручную
|
|
||||||
настроить iptables / межсетевой экран, чтобы разрешить доступ к Vitastor только с IP гипервизоров.
|
|
||||||
|
|
||||||
Также в этом случае нужно будет переключить обычные мосты на "Bridged & Security Groups" и включить фильтр
|
|
||||||
спуфинга IP в OpenNebula. Правда, реализация этого фильтра пока не полная, и она не блокирует доступ к
|
|
||||||
локальным интерфейсам гипервизора. То есть, включённый фильтр спуфинга IP запрещает ВМ отправлять трафик
|
|
||||||
с чужими IP к другим ВМ или во внешний мир, но не запрещает отправлять его напрямую гипервизору. Чтобы
|
|
||||||
исправить это, тоже нужны дополнительные правила iptables.
|
|
||||||
|
|
||||||
Таким образом, более-менее полная блокировка при использовании простой сети на сетевых мостах может
|
|
||||||
выглядеть так (здесь `10.0.3.0/24` - подсеть ВМ, `10.0.2.0/24` - подсеть гипервизора):
|
|
||||||
|
|
||||||
```
|
|
||||||
# Разрешаем входящий трафик с физического устройства
|
|
||||||
iptables -A INPUT -m physdev --physdev-in eth0 -j ACCEPT
|
|
||||||
# Запрещаем трафик со всех ВМ, но с IP не из подсети ВМ
|
|
||||||
iptables -A INPUT ! -s 10.0.3.0/24 -i onebr0 -j DROP
|
|
||||||
# Запрещаем трафик от ВМ к сети гипервизора
|
|
||||||
iptables -I FORWARD 1 -s 10.0.3.0/24 -d 10.0.2.0/24 -j DROP
|
|
||||||
```
|
|
||||||
|
|
||||||
## Тестирование
|
|
||||||
|
|
||||||
Плагин OpenNebula по большей части состоит из bash-скриптов, и чтобы было понятнее, что они
|
|
||||||
вообще делают - ниже приведены описания процедур, которыми можно протестировать каждый из них.
|
|
||||||
|
|
||||||
| Скрипт | Описание | Как протестировать |
|
|
||||||
| ----------------------- | --------------------------------------------- | ------------------------------------------------------------------------------------ |
|
|
||||||
| vmm/kvm/deploy.vitastor | Запустить виртуальную машину | Создайте и запустите виртуальную машину с дисками Vitastor: постоянным / непостоянным / волатильным (временным). |
|
|
||||||
| vmm/kvm/save.vitastor | Сохранить снимок памяти ВМ | Остановите виртуальную машину командой "Остановить". |
|
|
||||||
| vmm/kvm/restore.vitastor| Восстановить снимок памяти ВМ | Запустите ВМ после остановки обратно. |
|
|
||||||
| datastore/clone | Скопировать образ как "постоянный" | Создайте шаблон ВМ и создайте из него постоянную ВМ. |
|
|
||||||
| datastore/cp | Импортировать внешний образ | Импортируйте шаблон ВМ с образами дисков из Магазина OpenNebula. |
|
|
||||||
| datastore/export | Экспортировать образ как URL | Вероятно: экспортируйте шаблон ВМ с образами в Магазин. |
|
|
||||||
| datastore/mkfs | Создать образ с файловой системой | Хранилище → Образы → Создать → Тип: базовый блок данных, Расположение: пустой образ диска, Файловая система: любая непустая. |
|
|
||||||
| datastore/monitor | Вывод статистики места в хранилище образов | Проверьте статистику свободного/занятого места в списке хранилищ образов. |
|
|
||||||
| datastore/rm | Удалить "постоянный" образ | Хранилище → Образы → Выберите образ → Удалить. |
|
|
||||||
| datastore/snap_delete | Удалить снимок "постоянного" образа | Хранилище → Образы → Выберите образ → Выберите снимок → Удалить; <br> Чтобы создать образ со снимком: подключите постоянный образ к ВМ, создайте снимок, отключите образ. |
|
|
||||||
| datastore/snap_flatten | Откатить образ к снимку, удалив другие снимки | Хранилище → Образы → Выберите образ → Выберите снимок → "Выровнять" (flatten). |
|
|
||||||
| datastore/snap_revert | Откатить образ к снимку | Хранилище → Образы → Выберите образ → Выберите снимок → Откатить. |
|
|
||||||
| datastore/stat | Показать виртуальный размер образа в МБ | Неизвестно. По-видимому, в плагинах Vitastor и Ceph не используется. |
|
|
||||||
| tm/clone | Клонировать "непостоянный" образ в диск ВМ | Подключите "непостоянный" образ к ВМ. |
|
|
||||||
| tm/context | Создать диск контекстуализации ВМ | Создайте ВМ с контекстуализацией, как обычно. Но тестировать особенно нечего: в плагинах Vitastor и Ceph образ контекста хранится в локальной ФС гипервизора. |
|
|
||||||
| tm/cpds | Копировать диск ВМ/его снимок в новый образ | Выберите ВМ → Выберите диск → Опционально выберите снимок → "Сохранить как". |
|
|
||||||
| tm/delete | Удалить диск-клон или волатильный диск ВМ | Отключите волатильный или не-постоянный диск от ВМ. |
|
|
||||||
| tm/failmigrate | Обработать неудачную миграцию | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В других плагинах скрипт должен откатывать действия tm/premigrate. |
|
|
||||||
| tm/ln | Подключить "постоянный" образ к ВМ | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. |
|
|
||||||
| tm/mkimage | Создать волатильный диск, без или с ФС | Подключите волатильный диск к ВМ, с или без файловой системы. |
|
|
||||||
| tm/mkswap | Создать волатильный диск подкачки | Подключите волатильный диск к ВМ, форматированный как диск подкачки (swap). |
|
|
||||||
| tm/monitor | Вывод статистики места в хранилище дисков ВМ | Проверьте статистику свободного/занятого места в списке хранилищ дисков ВМ. |
|
|
||||||
| tm/mv | Мигрировать диск ВМ между хостами | Мигрируйте ВМ между серверами. Правда, с точки зрения хранилища в плагинах Vitastor и Ceph этот скрипт ничего не делает. |
|
|
||||||
| tm/mvds | Отключить "постоянный" образ от ВМ | Тестировать нечего. Скрипт пуст в плагинах Vitastor и Ceph. В целом же скрипт обратный к tm/ln и в других хранилищах он может, например, копировать образ ВМ с диска гипервизора обратно в хранилище. |
|
|
||||||
| tm/postbackup | Выполняется после бэкапа | По-видимому, скрипт просто удаляет временные файлы после резервного копирования. Так что можно провести его и проверить, что на серверах не осталось временных файлов. |
|
|
||||||
| tm/postbackup_live | Выполняется после бэкапа запущенной ВМ | То же, что tm/postbackup, но для запущенной ВМ. |
|
|
||||||
| tm/postmigrate | Выполняется после миграции ВМ | Тестировать нечего. Однако, OpenNebula запускает скрипт только для системного хранилища, поэтому он вызывает аналогичные скрипты для хранилищ других дисков той же ВМ. Помимо этого в плагинах Vitastor и Ceph скрипт ничего не делает. |
|
|
||||||
| tm/prebackup | Выполнить резервное копирование дисков ВМ | Создайте хранилище резервных копий типа "rsync" → Забэкапьте в него ВМ. |
|
|
||||||
| tm/prebackup_live | То же самое для запущенной ВМ | То же, что tm/prebackup, но запускает fsfreeze/thaw (остановку доступа к дискам). Так что смысл теста - проведите резервное копирование и проверьте, что данные скопировались консистентно. |
|
|
||||||
| tm/premigrate | Выполняется перед миграцией ВМ | Тестировать нечего. Аналогично tm/postmigrate запускается только для системного хранилища. |
|
|
||||||
| tm/resize | Изменить размер диска ВМ | Выберите ВМ → Выберите непостоянный диск → Измените его размер. |
|
|
||||||
| tm/restore | Восстановить диски ВМ из бэкапа | Создайте хранилище резервных копий → Забэкапьте в него ВМ → Восстановите её обратно. |
|
|
||||||
| tm/snap_create | Создать снимок диска ВМ | Выберите ВМ → Выберите диск → Создайте снимок. |
|
|
||||||
| tm/snap_create_live | Создать снимок диска запущенной ВМ | Выберите запущенную ВМ → Выберите диск → Создайте снимок. |
|
|
||||||
| tm/snap_delete | Удалить снимок диска ВМ | Выберите ВМ → Выберите диск → Выберите снимок → Удалить. |
|
|
||||||
| tm/snap_revert | Откатить диск ВМ к снимку | Выберите ВМ → Выберите диск → Выберите снимок → Откатить. |
|
|
|
@ -16,6 +16,8 @@
|
||||||
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
||||||
- Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
|
- Add `-oldstable` to bookworm/bullseye/buster in this line to install the last
|
||||||
stable version from 0.9.x branch instead of 1.x
|
stable version from 0.9.x branch instead of 1.x
|
||||||
|
- For Debian 10 (Buster) also enable backports repository:
|
||||||
|
`deb http://deb.debian.org/debian buster-backports main`
|
||||||
- Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
|
- Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
|
||||||
|
|
||||||
## CentOS
|
## CentOS
|
||||||
|
|
|
@ -16,6 +16,8 @@
|
||||||
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
- Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
|
||||||
- Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
|
- Добавьте `-oldstable` к слову bookworm/bullseye/buster в этой строке, чтобы
|
||||||
установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
|
установить последнюю стабильную версию из ветки 0.9.x вместо 1.x
|
||||||
|
- Для Debian 10 (Buster) также включите репозиторий backports:
|
||||||
|
`deb http://deb.debian.org/debian buster-backports main`
|
||||||
- Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
|
- Установите пакеты: `apt update; apt install vitastor lp-solve etcd linux-image-amd64 qemu-system-x86`
|
||||||
|
|
||||||
## CentOS
|
## CentOS
|
||||||
|
|
|
@ -17,10 +17,10 @@ To enable Vitastor support in Proxmox Virtual Environment (6.4-8.1 are supported
|
||||||
- Restart pvedaemon: `systemctl restart pvedaemon`
|
- Restart pvedaemon: `systemctl restart pvedaemon`
|
||||||
|
|
||||||
`/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others
|
`/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others
|
||||||
are listed below with their default values; `vitastor_ssd` is Proxmox storage pool id):
|
are listed below with their default values):
|
||||||
|
|
||||||
```
|
```
|
||||||
vitastor: vitastor_ssd
|
vitastor: vitastor
|
||||||
# pool to put new images into
|
# pool to put new images into
|
||||||
vitastor_pool testpool
|
vitastor_pool testpool
|
||||||
# path to the configuration file
|
# path to the configuration file
|
||||||
|
|
|
@ -16,10 +16,10 @@
|
||||||
- Перезапустите демон Proxmox: `systemctl restart pvedaemon`
|
- Перезапустите демон Proxmox: `systemctl restart pvedaemon`
|
||||||
|
|
||||||
Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные
|
Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные
|
||||||
перечислены внизу для понимания значений по умолчанию; `vitastor_ssd` - имя хранилища в Proxmox):
|
перечислены внизу для понимания значений по умолчанию):
|
||||||
|
|
||||||
```
|
```
|
||||||
vitastor: vitastor_ssd
|
vitastor: vitastor
|
||||||
# Пул, в который будут помещаться образы дисков
|
# Пул, в который будут помещаться образы дисков
|
||||||
vitastor_pool testpool
|
vitastor_pool testpool
|
||||||
# Путь к файлу конфигурации
|
# Путь к файлу конфигурации
|
||||||
|
|
|
@ -34,15 +34,9 @@
|
||||||
- [Client write-back cache](../config/client.en.md#client_enable_writeback)
|
- [Client write-back cache](../config/client.en.md#client_enable_writeback)
|
||||||
- [Intelligent recovery auto-tuning](../config/osd.en.md#recovery_tune_interval)
|
- [Intelligent recovery auto-tuning](../config/osd.en.md#recovery_tune_interval)
|
||||||
- [Clustered file system](../usage/nfs.en.md#vitastorfs)
|
- [Clustered file system](../usage/nfs.en.md#vitastorfs)
|
||||||
- [Experimental internal etcd replacement - antietcd](../config/monitor.en.md#use_antietcd)
|
|
||||||
- [Built-in Prometheus metric exporter](../config/monitor.en.md#enable_prometheus)
|
|
||||||
|
|
||||||
## Plugins and tools
|
## Plugins and tools
|
||||||
|
|
||||||
- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
|
|
||||||
- [OpenNebula storage plugin](../installation/opennebula.en.md)
|
|
||||||
- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
|
|
||||||
- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
|
|
||||||
- [Debian and CentOS packages](../installation/packages.en.md)
|
- [Debian and CentOS packages](../installation/packages.en.md)
|
||||||
- [Image management CLI (vitastor-cli)](../usage/cli.en.md)
|
- [Image management CLI (vitastor-cli)](../usage/cli.en.md)
|
||||||
- [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
|
- [Disk management CLI (vitastor-disk)](../usage/disk.en.md)
|
||||||
|
@ -50,6 +44,9 @@
|
||||||
- [Native QEMU driver](../usage/qemu.en.md)
|
- [Native QEMU driver](../usage/qemu.en.md)
|
||||||
- [Loadable fio engine for benchmarks](../usage/fio.en.md)
|
- [Loadable fio engine for benchmarks](../usage/fio.en.md)
|
||||||
- [NBD proxy for kernel mounts](../usage/nbd.en.md)
|
- [NBD proxy for kernel mounts](../usage/nbd.en.md)
|
||||||
|
- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
|
||||||
|
- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
|
||||||
|
- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
|
||||||
- [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md#pseudo-fs)
|
- [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md#pseudo-fs)
|
||||||
|
|
||||||
## Roadmap
|
## Roadmap
|
||||||
|
@ -59,6 +56,7 @@ The following features are planned for the future:
|
||||||
- Control plane optimisation
|
- Control plane optimisation
|
||||||
- Other administrative tools
|
- Other administrative tools
|
||||||
- Web GUI
|
- Web GUI
|
||||||
|
- OpenNebula plugin
|
||||||
- iSCSI and NVMeoF gateways
|
- iSCSI and NVMeoF gateways
|
||||||
- Multi-threaded client
|
- Multi-threaded client
|
||||||
- Faster failover
|
- Faster failover
|
||||||
|
|
|
@ -36,15 +36,9 @@
|
||||||
- [Буферизация записи на стороне клиента](../config/client.ru.md#client_enable_writeback)
|
- [Буферизация записи на стороне клиента](../config/client.ru.md#client_enable_writeback)
|
||||||
- [Интеллектуальная автоподстройка скорости восстановления](../config/osd.ru.md#recovery_tune_interval)
|
- [Интеллектуальная автоподстройка скорости восстановления](../config/osd.ru.md#recovery_tune_interval)
|
||||||
- [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
|
- [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
|
||||||
- [Экспериментальная встроенная замена etcd - antietcd](../config/monitor.ru.md#use_antietcd)
|
|
||||||
- [Встроенный Prometheus-экспортер метрик](../config/monitor.ru.md#enable_prometheus)
|
|
||||||
|
|
||||||
## Драйверы и инструменты
|
## Драйверы и инструменты
|
||||||
|
|
||||||
- [Плагин для Proxmox](../installation/proxmox.ru.md)
|
|
||||||
- [Плагин для OpenNebula](../installation/opennebula.ru.md)
|
|
||||||
- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
|
|
||||||
- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
|
|
||||||
- [Пакеты для Debian и CentOS](../installation/packages.ru.md)
|
- [Пакеты для Debian и CentOS](../installation/packages.ru.md)
|
||||||
- [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
|
- [Консольный интерфейс управления образами (vitastor-cli)](../usage/cli.ru.md)
|
||||||
- [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
|
- [Инструмент управления дисками (vitastor-disk)](../usage/disk.ru.md)
|
||||||
|
@ -52,6 +46,9 @@
|
||||||
- [Драйвер диска для QEMU](../usage/qemu.ru.md)
|
- [Драйвер диска для QEMU](../usage/qemu.ru.md)
|
||||||
- [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)
|
- [Драйвер диска для утилиты тестирования производительности fio](../usage/fio.ru.md)
|
||||||
- [NBD-прокси для монтирования образов ядром](../usage/nbd.ru.md) ("блочное устройство в режиме пользователя")
|
- [NBD-прокси для монтирования образов ядром](../usage/nbd.ru.md) ("блочное устройство в режиме пользователя")
|
||||||
|
- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
|
||||||
|
- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
|
||||||
|
- [Плагин для Proxmox](../installation/proxmox.ru.md)
|
||||||
- [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md#псевдо-фс)
|
- [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md#псевдо-фс)
|
||||||
|
|
||||||
## Планы развития
|
## Планы развития
|
||||||
|
@ -59,6 +56,7 @@
|
||||||
- Оптимизация слоя управления
|
- Оптимизация слоя управления
|
||||||
- Другие инструменты администрирования
|
- Другие инструменты администрирования
|
||||||
- Web-интерфейс
|
- Web-интерфейс
|
||||||
|
- Плагин для OpenNebula
|
||||||
- iSCSI и NVMeoF прокси
|
- iSCSI и NVMeoF прокси
|
||||||
- Многопоточный клиент
|
- Многопоточный клиент
|
||||||
- Более быстрое переключение при отказах
|
- Более быстрое переключение при отказах
|
||||||
|
|
|
@ -22,7 +22,7 @@
|
||||||
with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
|
with lazy fsync, but prepare for inferior single-thread latency. Read more about capacitors
|
||||||
[here](../config/layout-cluster.en.md#immediate_commit).
|
[here](../config/layout-cluster.en.md#immediate_commit).
|
||||||
- If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
|
- If you want to use HDDs, get modern HDDs with Media Cache or SSD Cache: HGST Ultrastar,
|
||||||
Toshiba MG, Seagate EXOS or something similar. If your drives don't have such cache then
|
Toshiba MG08, Seagate EXOS or something similar. If your drives don't have such cache then
|
||||||
you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
|
you also need small SSDs for journal and metadata (even 2 GB per 1 TB of HDD space is enough).
|
||||||
- Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
|
- Get a fast network (at least 10 Gbit/s). Something like Mellanox ConnectX-4 with RoCEv2 is ideal.
|
||||||
- Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
|
- Disable CPU powersaving: `cpupower idle-set -D 0 && cpupower frequency-set -g performance`.
|
||||||
|
@ -33,7 +33,7 @@
|
||||||
- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
|
- SATA SSD: Micron 5100/5200/5300/5400, Samsung PM863/PM883/PM893, Intel D3-S4510/4520/4610/4620, Kingston DC500M
|
||||||
- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
|
- NVMe: Micron 9100/9200/9300/9400, Micron 7300/7450, Samsung PM983/PM9A3, Samsung PM1723/1735/1743,
|
||||||
Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
|
Intel DC-P3700/P4500/P4600, Intel D7-P5500/P5600, Intel Optane, Kingston DC1000B/DC1500M
|
||||||
- HDD: HGST Ultrastar, Toshiba MG, Seagate EXOS
|
- HDD: HGST Ultrastar, Toshiba MG06/MG07/MG08, Seagate EXOS
|
||||||
|
|
||||||
## Configure monitors
|
## Configure monitors
|
||||||
|
|
||||||
|
@ -68,6 +68,10 @@ On the monitor hosts:
|
||||||
but some free unpartitioned space must be available because the script creates new partitions for journals.
|
but some free unpartitioned space must be available because the script creates new partitions for journals.
|
||||||
- You can change OSD configuration in units or in `vitastor.conf`.
|
- You can change OSD configuration in units or in `vitastor.conf`.
|
||||||
Check [Configuration Reference](../config.en.md) for parameter descriptions.
|
Check [Configuration Reference](../config.en.md) for parameter descriptions.
|
||||||
|
- If all your drives have capacitors, and even if not, but if you ran `vitastor-disk`
|
||||||
|
without `--disable_data_fsync off` at the first step, then put the following
|
||||||
|
setting into etcd: \
|
||||||
|
`etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
|
||||||
- Start all OSDs: `systemctl start vitastor.target`
|
- Start all OSDs: `systemctl start vitastor.target`
|
||||||
|
|
||||||
## Create a pool
|
## Create a pool
|
||||||
|
@ -84,10 +88,6 @@ For EC pools the configuration should look like the following:
|
||||||
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
||||||
```
|
```
|
||||||
|
|
||||||
Add `--immediate_commit none` if you added `--disable_data_fsync off` at the OSD
|
|
||||||
initialization step, or if `vitastor-disk` complained about impossibility to
|
|
||||||
disable drive cache.
|
|
||||||
|
|
||||||
After you do this, one of the monitors will configure PGs and OSDs will start them.
|
After you do this, one of the monitors will configure PGs and OSDs will start them.
|
||||||
|
|
||||||
If you use HDDs you should also add `"block_size": 1048576` to pool configuration.
|
If you use HDDs you should also add `"block_size": 1048576` to pool configuration.
|
||||||
|
|
|
@ -69,6 +69,11 @@
|
||||||
для журналов, на SSD должно быть доступно свободное нераспределённое место.
|
для журналов, на SSD должно быть доступно свободное нераспределённое место.
|
||||||
- Вы можете менять параметры OSD в юнитах systemd или в `vitastor.conf`. Описания параметров
|
- Вы можете менять параметры OSD в юнитах systemd или в `vitastor.conf`. Описания параметров
|
||||||
смотрите в [справке по конфигурации](../config.ru.md).
|
смотрите в [справке по конфигурации](../config.ru.md).
|
||||||
|
- Если все ваши диски - серверные с конденсаторами, и даже если нет, но при этом
|
||||||
|
вы не добавляли опцию `--disable_data_fsync off` на первом шаге, а `vitastor-disk`
|
||||||
|
не ругался на невозможность отключения кэша дисков, пропишите следующую настройку
|
||||||
|
в глобальную конфигурацию в etcd: \
|
||||||
|
`etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`.
|
||||||
- Запустите все OSD: `systemctl start vitastor.target`
|
- Запустите все OSD: `systemctl start vitastor.target`
|
||||||
|
|
||||||
## Создайте пул
|
## Создайте пул
|
||||||
|
@ -85,10 +90,6 @@ vitastor-cli create-pool testpool --pg_size 2 --pg_count 256
|
||||||
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
||||||
```
|
```
|
||||||
|
|
||||||
Добавьте также опцию `--immediate_commit none`, если вы добавляли `--disable_data_fsync off`
|
|
||||||
на этапе инициализации OSD, либо если `vitastor-disk` ругался на невозможность отключения
|
|
||||||
кэша дисков.
|
|
||||||
|
|
||||||
После этого один из мониторов должен сконфигурировать PG, а OSD должны запустить их.
|
После этого один из мониторов должен сконфигурировать PG, а OSD должны запустить их.
|
||||||
|
|
||||||
Если вы используете HDD-диски, то добавьте в конфигурацию пулов опцию `"block_size": 1048576`.
|
Если вы используете HDD-диски, то добавьте в конфигурацию пулов опцию `"block_size": 1048576`.
|
||||||
|
@ -122,4 +123,4 @@ vitastor-cli create -s 10G testimg
|
||||||
Если вы хотите использовать не только блочные образы виртуальных машин или контейнеров,
|
Если вы хотите использовать не только блочные образы виртуальных машин или контейнеров,
|
||||||
а также кластерную файловую систему, то:
|
а также кластерную файловую систему, то:
|
||||||
|
|
||||||
- [Следуйте инструкциям](../usage/nfs.ru.md#vitastorfs)
|
- [Следуйте инструкциям](../usage/nfs.en.md#vitastorfs)
|
||||||
|
|
|
@ -42,7 +42,7 @@ PG state always includes exactly 1 of the following base states:
|
||||||
- **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for
|
- **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for
|
||||||
this PG at all (if the pool is just created), or an unavailable OSD is set as primary,
|
this PG at all (if the pool is just created), or an unavailable OSD is set as primary,
|
||||||
or the primary OSD refuses to start this PG (for example, because of wrong block_size),
|
or the primary OSD refuses to start this PG (for example, because of wrong block_size),
|
||||||
or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/pg/config` in etcd.
|
or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/config/pgs` in etcd.
|
||||||
- **starting** — primary OSD has acquired PG lock in etcd, PG is starting.
|
- **starting** — primary OSD has acquired PG lock in etcd, PG is starting.
|
||||||
- **peering** — primary OSD requests PG object listings from secondary OSDs and calculates
|
- **peering** — primary OSD requests PG object listings from secondary OSDs and calculates
|
||||||
the PG state.
|
the PG state.
|
||||||
|
@ -107,17 +107,16 @@ If a PG is active it can also have any number of the following additional states
|
||||||
|
|
||||||
## Removing a healthy disk
|
## Removing a healthy disk
|
||||||
|
|
||||||
Before removing a healthy disk from the cluster set its OSD weight(s) to 0 to
|
Befor removing a healthy disk from the cluster set its OSD weight(s) to 0 to
|
||||||
move data away. To do that, run `vitastor-cli modify-osd --reweight 0 <НОМЕР_OSD>`.
|
move data away. To do that, add `"reweight":0` to etcd key `/vitastor/config/osd/<OSD_NUMBER>`.
|
||||||
|
For example:
|
||||||
Then wait until rebalance finishes and remove OSD by running `vitastor-disk purge /dev/vitastor/osdN-data`.
|
|
||||||
|
|
||||||
Zero weight can also be put manually into etcd key `/vitastor/config/osd/<НОМЕР_OSD>`, for example:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Then wait until rebalance finishes and remove OSD by running `vitastor-disk purge /dev/vitastor/osdN-data`.
|
||||||
|
|
||||||
## Removing a failed disk
|
## Removing a failed disk
|
||||||
|
|
||||||
If a disk is already dead, its OSD(s) are likely already stopped.
|
If a disk is already dead, its OSD(s) are likely already stopped.
|
||||||
|
@ -150,7 +149,7 @@ POOL_ID=1
|
||||||
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
||||||
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
||||||
for i in $(seq 1 $PG_COUNT); do
|
for i in $(seq 1 $PG_COUNT); do
|
||||||
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'
|
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'; done
|
||||||
done
|
done
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -169,63 +168,21 @@ Upgrading is performed without stopping clients (VMs/containers), you just need
|
||||||
upgrade and restart servers one by one. However, ideally you should restart VMs too
|
upgrade and restart servers one by one. However, ideally you should restart VMs too
|
||||||
to make them use the new version of the client library.
|
to make them use the new version of the client library.
|
||||||
|
|
||||||
### 1.7.x to 1.8.0
|
Exceptions (specific upgrade instructions):
|
||||||
|
- Upgrading <= 1.1.x to 1.2.0 or later, if you use EC n+k with k>=2, is recommended
|
||||||
|
to be performed with full downtime: first you should stop all clients, then all OSDs,
|
||||||
|
then upgrade and start everything back — because versions before 1.2.0 have several
|
||||||
|
bugs leading to invalid data being read in EC n+k, k>=2 configurations in degraded pools.
|
||||||
|
- Versions <= 0.8.7 are incompatible with versions >= 0.9.0, so you should first
|
||||||
|
upgrade from <= 0.8.7 to 0.8.8 or 0.8.9, and only then to >= 0.9.x. If you upgrade
|
||||||
|
without this intermediate step, client I/O will hang until the end of upgrade process.
|
||||||
|
- Upgrading from <= 0.5.x to >= 0.6.x is not supported.
|
||||||
|
|
||||||
After upgrading version <= 1.7.x to version >= 1.8.0, BUT <= 1.9.0: restart all clients
|
Rollback:
|
||||||
(VMs and so on), otherwise they will hang when monitor clears old PG configuration key,
|
- Version 1.0.0 has a new disk format, so OSDs initiaziled on 1.0.0 can't be rolled
|
||||||
which happens 24 hours after upgrade.
|
back to 0.9.x or previous versions.
|
||||||
|
- Versions before 0.8.0 don't have vitastor-disk, so OSDs, initialized by it, won't
|
||||||
This is fixed in 1.9.1. So, after upgrading version <= 1.7.x directly to version >= 1.9.1,
|
start with 0.7.x or 0.6.x. :-)
|
||||||
you DO NOT have to restart all old clients immediately - they will work like before until
|
|
||||||
you decide to upgrade them too. The downside is that you'll have to remove the old PG
|
|
||||||
configuration key (`/vitastor/config/pgs`) from etcd by hand when you make sure that all
|
|
||||||
your clients are restarted.
|
|
||||||
|
|
||||||
### 1.1.x to 1.2.0
|
|
||||||
|
|
||||||
Upgrading version <= 1.1.x to version >= 1.2.0, if you use EC n+k with k>=2, is recommended
|
|
||||||
to be performed with full downtime: first you should stop all clients, then all OSDs,
|
|
||||||
then upgrade and start everything back — because versions before 1.2.0 have several
|
|
||||||
bugs leading to invalid data being read in EC n+k, k>=2 configurations in degraded pools.
|
|
||||||
|
|
||||||
### 0.8.7 to 0.9.0
|
|
||||||
|
|
||||||
Versions <= 0.8.7 are incompatible with versions >= 0.9.0, so you should first
|
|
||||||
upgrade from <= 0.8.7 to 0.8.8 or 0.8.9, and only then to >= 0.9.x. If you upgrade
|
|
||||||
without this intermediate step, client I/O will hang until the end of upgrade process.
|
|
||||||
|
|
||||||
### 0.5.x to 0.6.x
|
|
||||||
|
|
||||||
Upgrading from <= 0.5.x to >= 0.6.x is not supported.
|
|
||||||
|
|
||||||
## Downgrade
|
|
||||||
|
|
||||||
Downgrade are also allowed freely, except the following specific instructions:
|
|
||||||
|
|
||||||
### 1.8.0 to 1.7.1
|
|
||||||
|
|
||||||
Before downgrading from version >= 1.8.0 to version <= 1.7.1
|
|
||||||
you have to copy /vitastor/pg/config etcd key to /vitastor/config/pgs:
|
|
||||||
|
|
||||||
```
|
|
||||||
etcdctl --endpoints=http://... get --print-value-only /vitastor/pg/config | \
|
|
||||||
etcdctl --endpoints=http://... put /vitastor/config/pgs
|
|
||||||
```
|
|
||||||
|
|
||||||
Then you can just install older packages and restart all services.
|
|
||||||
|
|
||||||
If you performed downgrade without first copying that key, run "add all OSDs into the
|
|
||||||
history records of all PGs" from [Restoring from lost pool configuration](#restoring-from-lost-pool-configuration).
|
|
||||||
|
|
||||||
### 1.0.0 to 0.9.x
|
|
||||||
|
|
||||||
Version 1.0.0 has a new disk format, so OSDs initialized on 1.0.0 or later can't
|
|
||||||
be rolled back to 0.9.x or previous versions.
|
|
||||||
|
|
||||||
### 0.8.0 to 0.7.x
|
|
||||||
|
|
||||||
Versions before 0.8.0 don't have vitastor-disk, so OSDs, initialized by it, won't
|
|
||||||
start with older versions (0.4.x - 0.7.x). :-)
|
|
||||||
|
|
||||||
## OSD memory usage
|
## OSD memory usage
|
||||||
|
|
||||||
|
|
|
@ -42,7 +42,7 @@
|
||||||
- **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще
|
- **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще
|
||||||
(если пул только создан), либо в качестве первичного назначен недоступный OSD, либо
|
(если пул только создан), либо в качестве первичного назначен недоступный OSD, либо
|
||||||
назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size),
|
назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size),
|
||||||
либо PG остановлена монитором через флаг `pause: true` в `/vitastor/pg/config` в etcd.
|
либо PG остановлена монитором через флаг `pause: true` в `/vitastor/config/pgs` в etcd.
|
||||||
- **starting** — первичный OSD захватил блокировку PG в etcd, PG запускается.
|
- **starting** — первичный OSD захватил блокировку PG в etcd, PG запускается.
|
||||||
- **peering** — первичный OSD опрашивает вторичные OSD на предмет списков объектов данной PG и рассчитывает её состояние.
|
- **peering** — первичный OSD опрашивает вторичные OSD на предмет списков объектов данной PG и рассчитывает её состояние.
|
||||||
- **repeering** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **peering**.
|
- **repeering** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **peering**.
|
||||||
|
@ -105,16 +105,14 @@ PG должны очень быстро переходить из них в др
|
||||||
## Удаление исправного диска
|
## Удаление исправного диска
|
||||||
|
|
||||||
Перед удалением исправного диска из кластера установите его OSD вес в 0, чтобы убрать с него данные.
|
Перед удалением исправного диска из кластера установите его OSD вес в 0, чтобы убрать с него данные.
|
||||||
Для этого выполните команду `vitastor-cli modify-osd --reweight 0 <НОМЕР_OSD>`.
|
Для этого добавьте в ключ `/vitastor/config/osd/<НОМЕР_OSD>` в etcd значение `"reweight":0`, например:
|
||||||
|
|
||||||
Дождитесь завершения перебалансировки данных, после чего удалите OSD командой `vitastor-disk purge /dev/vitastor/osdN-data`.
|
|
||||||
|
|
||||||
Также вес 0 можно прописать вручную прямо в etcd в ключ `/vitastor/config/osd/<НОМЕР_OSD>`, например:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Дождитесь завершения ребаланса, после чего удалите OSD командой `vitastor-disk purge /dev/vitastor/osdN-data`.
|
||||||
|
|
||||||
## Удаление неисправного диска
|
## Удаление неисправного диска
|
||||||
|
|
||||||
Если диск уже умер, его OSD, скорее всего, уже будет/будут остановлен(ы).
|
Если диск уже умер, его OSD, скорее всего, уже будет/будут остановлен(ы).
|
||||||
|
@ -147,7 +145,7 @@ POOL_ID=1
|
||||||
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
||||||
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
||||||
for i in $(seq 1 $PG_COUNT); do
|
for i in $(seq 1 $PG_COUNT); do
|
||||||
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'
|
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'; done
|
||||||
done
|
done
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -166,63 +164,21 @@ done
|
||||||
достаточно обновлять серверы по одному. Однако, конечно, чтобы запущенные виртуальные машины
|
достаточно обновлять серверы по одному. Однако, конечно, чтобы запущенные виртуальные машины
|
||||||
начали использовать новую версию клиентской библиотеки, их тоже нужно перезапустить.
|
начали использовать новую версию клиентской библиотеки, их тоже нужно перезапустить.
|
||||||
|
|
||||||
### 1.7.x -> 1.8.0
|
Исключения (особые указания при обновлении):
|
||||||
|
- Обновляться с версий <= 1.1.x до версий >= 1.2.0, если вы используете EC n+k и k>=2,
|
||||||
|
рекомендуется с временной остановкой кластера — сначала нужно остановить всех клиентов,
|
||||||
|
потом все OSD, потом обновить и запустить всё обратно — из-за нескольких багов, которые
|
||||||
|
могли приводить к некорректному чтению данных в деградированных EC-пулах.
|
||||||
|
- Версии <= 0.8.7 несовместимы с версиями >= 0.9.0, поэтому при обновлении с <= 0.8.7
|
||||||
|
нужно сначала обновиться до 0.8.8 или 0.8.9, а уже потом до любых версий >= 0.9.x.
|
||||||
|
Иначе клиентский ввод-вывод зависнет до завершения обновления.
|
||||||
|
- Обновление с версий 0.5.x и более ранних до 0.6.x и более поздних не поддерживается.
|
||||||
|
|
||||||
После обновления с версий <= 1.7.x до версий >= 1.8.0, НО <= 1.9.0: перезапустите всех
|
Откат:
|
||||||
клиентов (процессы виртуальных машин можно перезапустить путём миграции на другой сервер),
|
- В версии 1.0.0 поменялся дисковый формат, поэтому OSD, созданные на версии >= 1.0.0,
|
||||||
иначе они зависнут, когда монитор удалит старый ключ конфигурации PG, что происходит через
|
нельзя откатить до версии 0.9.x и более ранних.
|
||||||
24 часа после обновления.
|
- В версиях ранее 0.8.0 нет vitastor-disk, значит, созданные им OSD нельзя откатить
|
||||||
|
до 0.7.x или 0.6.x. :-)
|
||||||
Однако, это исправлено в 1.9.1. Так что, если вы обновляетесь с <= 1.7.x сразу до >= 1.9.1,
|
|
||||||
вам НЕ нужно сразу перезапускать всех клиентов - они будут работать, как раньше. Минус,
|
|
||||||
правда, в том, что старый ключ конфигурации PG (`/vitastor/config/pgs`) будет нужно удалить
|
|
||||||
вам из etcd вручную - после того, как вы убедитесь, что все клиенты перезапущены.
|
|
||||||
|
|
||||||
### 1.1.x -> 1.2.0
|
|
||||||
|
|
||||||
Обновляться с версий <= 1.1.x до версий >= 1.2.0, если вы используете EC n+k и k>=2,
|
|
||||||
рекомендуется с временной остановкой кластера — сначала нужно остановить всех клиентов,
|
|
||||||
потом все OSD, потом обновить и запустить всё обратно — из-за нескольких багов, которые
|
|
||||||
могли приводить к некорректному чтению данных в деградированных EC-пулах.
|
|
||||||
|
|
||||||
### 0.8.7 -> 0.9.0
|
|
||||||
|
|
||||||
Версии <= 0.8.7 несовместимы с версиями >= 0.9.0, поэтому при обновлении с <= 0.8.7
|
|
||||||
нужно сначала обновиться до 0.8.8 или 0.8.9, а уже потом до любых версий >= 0.9.x.
|
|
||||||
Иначе клиентский ввод-вывод зависнет до завершения обновления.
|
|
||||||
|
|
||||||
### 0.5.x -> 0.6.x
|
|
||||||
|
|
||||||
Обновление с версий 0.5.x и более ранних до 0.6.x и более поздних не поддерживается.
|
|
||||||
|
|
||||||
## Откат версии
|
|
||||||
|
|
||||||
Откат (понижение версии) тоже свободно разрешён, кроме указанных ниже случаев:
|
|
||||||
|
|
||||||
### 1.8.0 -> 1.7.1
|
|
||||||
|
|
||||||
Перед понижением версии с >= 1.8.0 до <= 1.7.1 вы должны скопировать ключ
|
|
||||||
etcd `/vitastor/pg/config` в `/vitastor/config/pgs`:
|
|
||||||
|
|
||||||
```
|
|
||||||
etcdctl --endpoints=http://... get --print-value-only /vitastor/pg/config | \
|
|
||||||
etcdctl --endpoints=http://... put /vitastor/config/pgs
|
|
||||||
```
|
|
||||||
|
|
||||||
После этого можно просто установить более старые пакеты и перезапустить все сервисы.
|
|
||||||
|
|
||||||
Если вы откатили версию, не скопировав предварительно этот ключ - выполните "добавление всех
|
|
||||||
OSD в исторические записи всех PG" из раздела [Восстановление потерянной конфигурации пулов](#восстановление-потерянной-конфигурации-пулов).
|
|
||||||
|
|
||||||
### 1.0.0 -> 0.9.x
|
|
||||||
|
|
||||||
В версии 1.0.0 поменялся дисковый формат, поэтому OSD, созданные на версии >= 1.0.0,
|
|
||||||
нельзя откатить до версии 0.9.x и более ранних.
|
|
||||||
|
|
||||||
### 0.8.0 -> 0.7.x
|
|
||||||
|
|
||||||
В версиях ранее 0.8.0 нет vitastor-disk, значит, созданные им OSD не запустятся на
|
|
||||||
более ранних версиях (0.4.x - 0.7.x). :-)
|
|
||||||
|
|
||||||
## Потребление памяти OSD
|
## Потребление памяти OSD
|
||||||
|
|
||||||
|
|
|
@ -16,7 +16,6 @@ It supports the following commands:
|
||||||
- [create](#create)
|
- [create](#create)
|
||||||
- [snap-create](#create)
|
- [snap-create](#create)
|
||||||
- [modify](#modify)
|
- [modify](#modify)
|
||||||
- [dd](#dd)
|
|
||||||
- [rm](#rm)
|
- [rm](#rm)
|
||||||
- [flatten](#flatten)
|
- [flatten](#flatten)
|
||||||
- [rm-data](#rm-data)
|
- [rm-data](#rm-data)
|
||||||
|
@ -25,10 +24,6 @@ It supports the following commands:
|
||||||
- [fix](#fix)
|
- [fix](#fix)
|
||||||
- [alloc-osd](#alloc-osd)
|
- [alloc-osd](#alloc-osd)
|
||||||
- [rm-osd](#rm-osd)
|
- [rm-osd](#rm-osd)
|
||||||
- [osd-tree](#osd-tree)
|
|
||||||
- [ls-osd](#ls-osd)
|
|
||||||
- [modify-osd](#modify-osd)
|
|
||||||
- [pg-list](#pg-list)
|
|
||||||
- [create-pool](#create-pool)
|
- [create-pool](#create-pool)
|
||||||
- [modify-pool](#modify-pool)
|
- [modify-pool](#modify-pool)
|
||||||
- [ls-pools](#ls-pools)
|
- [ls-pools](#ls-pools)
|
||||||
|
@ -149,60 +144,19 @@ You should resize file system in the image, if present, before shrinking it.
|
||||||
* `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
|
* `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
|
||||||
* `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
|
* `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
|
||||||
|
|
||||||
## dd
|
|
||||||
|
|
||||||
```
|
|
||||||
vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
|
|
||||||
[count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
|
|
||||||
[conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
|
|
||||||
```
|
|
||||||
|
|
||||||
Copy data between Vitastor images, files and pipes.
|
|
||||||
|
|
||||||
Options can be specified in classic dd style (`key=value`) or like usual (`--key value`).
|
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
|
||||||
|-----------------|-------------------------------------------------------------------------|
|
|
||||||
| `iimg=<image>` | Copy from Vitastor image `<image>` |
|
|
||||||
| `if=<file>` | Copy from file `<file>` |
|
|
||||||
| `oimg=<image>` | Copy to Vitastor image `<image>` |
|
|
||||||
| `of=<file>` | Copy to file `<file>` |
|
|
||||||
| `bs=1M` | Set copy block size |
|
|
||||||
| `count=N` | Copy only N input blocks. If N ends in B it counts bytes, not blocks |
|
|
||||||
| `seek/oseek=N` | Skip N output blocks. If N ends in B it counts bytes, not blocks |
|
|
||||||
| `skip/iseek=N` | Skip N input blocks. If N ends in B it counts bytes, not blocks |
|
|
||||||
| `iodepth=N` | Send N reads or writes in parallel (default 4) |
|
|
||||||
| `status=LEVEL` | The LEVEL of information to print to stderr: none/noxfer/progress |
|
|
||||||
| `size=N` | Specify size for the created output file/image (defaults to input size) |
|
|
||||||
| `iflag=direct` | For input files only: use direct I/O |
|
|
||||||
| `oflag=direct` | For output files only: use direct I/O |
|
|
||||||
| `oflag=append` | For files only: append to output file |
|
|
||||||
| `conv=nocreat` | Do not create output file/image |
|
|
||||||
| `conv=trunc` | Truncate output file/image |
|
|
||||||
| `conv=noerror` | Continue copying after errors |
|
|
||||||
| `conv=nofsync` | Do not call fsync before finishing (default behaviour is fsync) |
|
|
||||||
| `conv=nosparse` | Write all output blocks including all-zero blocks |
|
|
||||||
|
|
||||||
## rm
|
## rm
|
||||||
|
|
||||||
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
||||||
|
|
||||||
`vitastor-cli rm (--exact|--matching) <glob> ...`
|
Remove `<from>` or all layers between `<from>` and `<to>` (`<to>` must be a child of `<from>`),
|
||||||
|
rebasing all their children accordingly. --writers-stopped allows merging to be a bit
|
||||||
|
more effective in case of a single 'slim' read-write child and 'fat' removed parent:
|
||||||
|
the child is merged into parent and parent is renamed to child in that case.
|
||||||
|
In other cases parent layers are always merged into children.
|
||||||
|
|
||||||
Remove layer(s) and rebase all their children accordingly.
|
Other options:
|
||||||
|
|
||||||
In the first form, remove `<from>` or layers between `<from>` and its child `<to>`.
|
* `--down-ok` - Continue deletion/merging even if some data will be left on unavailable OSDs.
|
||||||
|
|
||||||
In the second form, remove all images with exact or pattern-matched names.
|
|
||||||
|
|
||||||
Options:
|
|
||||||
|
|
||||||
* `--writers-stopped` allows optimised removal in case of a single 'slim' read-write
|
|
||||||
child and 'fat' removed parent: the child is merged into parent and parent is renamed
|
|
||||||
to child in that case. In other cases parent layers are always merged into children.
|
|
||||||
* `--exact` - remove multiple images with names matching given glob patterns.
|
|
||||||
* `--matching` - remove multiple images with given names
|
|
||||||
* `--down-ok` - continue deletion/merging even if some data will be left on unavailable OSDs.
|
|
||||||
|
|
||||||
## flatten
|
## flatten
|
||||||
|
|
||||||
|
@ -220,7 +174,6 @@ Remove inode data without changing metadata.
|
||||||
--wait-list Retrieve full objects listings before starting to remove objects.
|
--wait-list Retrieve full objects listings before starting to remove objects.
|
||||||
Requires more memory, but allows to show correct removal progress.
|
Requires more memory, but allows to show correct removal progress.
|
||||||
--min-offset Purge only data starting with specified offset.
|
--min-offset Purge only data starting with specified offset.
|
||||||
--max-offset Purge only data before specified offset.
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## merge-data
|
## merge-data
|
||||||
|
@ -293,82 +246,6 @@ Refuses to remove OSDs with data without `--force` and `--allow-data-loss`.
|
||||||
With `--dry-run` only checks if deletion is possible without data loss and
|
With `--dry-run` only checks if deletion is possible without data loss and
|
||||||
redundancy degradation.
|
redundancy degradation.
|
||||||
|
|
||||||
## osd-tree
|
|
||||||
|
|
||||||
`vitastor-cli osd-tree [-l|--long]`
|
|
||||||
|
|
||||||
Show current OSD tree, optionally with I/O statistics if -l is specified.
|
|
||||||
|
|
||||||
Example output:
|
|
||||||
|
|
||||||
```
|
|
||||||
TYPE NAME UP SIZE USED% TAGS WEIGHT BLOCK BITMAP IMM NOOUT
|
|
||||||
host kaveri
|
|
||||||
disk nvme0n1p1
|
|
||||||
osd 3 down 100G 0 % abc,kaveri 1 128k 4k none -
|
|
||||||
osd 4 down 100G 0 % 1 128k 4k none -
|
|
||||||
disk nvme1n1p1
|
|
||||||
osd 5 down 100G 0 % abc,kaveri 1 128k 4k none -
|
|
||||||
osd 6 down 100G 0 % 1 128k 4k none -
|
|
||||||
host stump
|
|
||||||
osd 1 up 100G 37.29 % osdone 1 128k 4k all -
|
|
||||||
osd 2 up 100G 26.8 % abc 1 128k 4k all -
|
|
||||||
osd 7 up 100G 21.84 % 1 128k 4k all -
|
|
||||||
osd 8 up 100G 21.63 % 1 128k 4k all -
|
|
||||||
osd 9 up 100G 20.69 % 1 128k 4k all -
|
|
||||||
osd 10 up 100G 21.61 % 1 128k 4k all -
|
|
||||||
osd 11 up 100G 21.53 % 1 128k 4k all -
|
|
||||||
osd 12 up 100G 22.4 % 1 128k 4k all -
|
|
||||||
```
|
|
||||||
|
|
||||||
## ls-osd
|
|
||||||
|
|
||||||
`vitastor-cli osds|ls-osd|osd-ls [-l|--long]`
|
|
||||||
|
|
||||||
Show current OSDs as list, optionally with I/O statistics if -l is specified.
|
|
||||||
|
|
||||||
Example output:
|
|
||||||
|
|
||||||
```
|
|
||||||
OSD PARENT UP SIZE USED% TAGS WEIGHT BLOCK BITMAP IMM NOOUT
|
|
||||||
3 kaveri/nvme0n1p1 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
4 kaveri/nvme0n1p1 down 100G 0 % 1 128k 4k none -
|
|
||||||
5 kaveri/nvme1n1p1 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
6 kaveri/nvme1n1p1 down 100G 0 % 1 128k 4k none -
|
|
||||||
1 stump up 100G 37.29 % osdone 1 128k 4k all -
|
|
||||||
2 stump up 100G 26.8 % globl 1 128k 4k all -
|
|
||||||
7 stump up 100G 21.84 % 1 128k 4k all -
|
|
||||||
8 stump up 100G 21.63 % 1 128k 4k all -
|
|
||||||
9 stump up 100G 20.69 % 1 128k 4k all -
|
|
||||||
10 stump up 100G 21.61 % 1 128k 4k all -
|
|
||||||
11 stump up 100G 21.53 % 1 128k 4k all -
|
|
||||||
12 stump up 100G 22.4 % 1 128k 4k all -
|
|
||||||
```
|
|
||||||
|
|
||||||
## modify-osd
|
|
||||||
|
|
||||||
`vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
|
|
||||||
|
|
||||||
Set OSD reweight, tags or noout flag. See detail description in [OSD config documentation](../config/pool.en.md#osd-settings).
|
|
||||||
|
|
||||||
## pg-list
|
|
||||||
|
|
||||||
`vitastor-cli pg-list|pg-ls|list-pg|ls-pg|ls-pgs [OPTIONS] [state1+state2] [^state3] [...]`
|
|
||||||
|
|
||||||
List PGs with any of listed state filters (^ or ! in the beginning is negation). Options:
|
|
||||||
|
|
||||||
```
|
|
||||||
--pool <pool name or number> Only list PGs of the given pool.
|
|
||||||
--min <min pg number> Only list PGs with number >= min.
|
|
||||||
--max <max pg number> Only list PGs with number <= max.
|
|
||||||
```
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
|
|
||||||
`vitastor-cli pg-list active+degraded`
|
|
||||||
|
|
||||||
`vitastor-cli pg-list ^active`
|
|
||||||
|
|
||||||
## create-pool
|
## create-pool
|
||||||
|
|
||||||
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
||||||
|
|
|
@ -17,17 +17,12 @@ vitastor-cli - интерфейс командной строки для адм
|
||||||
- [create](#create)
|
- [create](#create)
|
||||||
- [snap-create](#create)
|
- [snap-create](#create)
|
||||||
- [modify](#modify)
|
- [modify](#modify)
|
||||||
- [dd](#dd)
|
|
||||||
- [rm](#rm)
|
- [rm](#rm)
|
||||||
- [flatten](#flatten)
|
- [flatten](#flatten)
|
||||||
- [rm-data](#rm-data)
|
- [rm-data](#rm-data)
|
||||||
- [merge-data](#merge-data)
|
- [merge-data](#merge-data)
|
||||||
- [alloc-osd](#alloc-osd)
|
- [alloc-osd](#alloc-osd)
|
||||||
- [rm-osd](#rm-osd)
|
- [rm-osd](#rm-osd)
|
||||||
- [osd-tree](#osd-tree)
|
|
||||||
- [ls-osd](#ls-osd)
|
|
||||||
- [modify-osd](#modify-osd)
|
|
||||||
- [pg-list](#pg-list)
|
|
||||||
- [create-pool](#create-pool)
|
- [create-pool](#create-pool)
|
||||||
- [modify-pool](#modify-pool)
|
- [modify-pool](#modify-pool)
|
||||||
- [ls-pools](#ls-pools)
|
- [ls-pools](#ls-pools)
|
||||||
|
@ -152,61 +147,23 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
|
||||||
* `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
|
* `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
|
||||||
* `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
|
* `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
|
||||||
|
|
||||||
## dd
|
|
||||||
|
|
||||||
```
|
|
||||||
vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
|
|
||||||
[count=N] [seek/oseek=N] [skip/iseek=M] [iodepth=N] [status=progress] \
|
|
||||||
[conv=nocreat,noerror,nofsync,trunc,nosparse] [iflag=direct] [oflag=direct,append]
|
|
||||||
```
|
|
||||||
|
|
||||||
Копировать данные между образами Vitastor, файлами и каналами.
|
|
||||||
|
|
||||||
Опции можно передавать в классическом стиле dd (`key=value`) или как обычно (`--key value`).
|
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
|
||||||
|-----------------|-------------------------------------------------------------------------|
|
|
||||||
| `iimg=<image>` | Копировать из образа Vitastor `<image>` |
|
|
||||||
| `if=<file>` | Копировать из файла `<file>` |
|
|
||||||
| `oimg=<image>` | Копировать в образ Vitastor `<image>` |
|
|
||||||
| `of=<file>` | Копировать в файл `<file>` |
|
|
||||||
| `bs=1M` | Задать размер блока копирования |
|
|
||||||
| `count=N` | Копировать не более N блоков. Если N заканчивается на B - то N байт. |
|
|
||||||
| `seek/oseek=N` | Пропустить N выходных блоков. Если N заканчивается на B - то N байт. |
|
|
||||||
| `skip/iseek=N` | Пропустить N входных блоков. Если N заканчивается на B - то N байт. |
|
|
||||||
| `iodepth=N` | Отправлять N чтений/записей параллельно (по умолчанию 4). |
|
|
||||||
| `status=LEVEL` | Уровень вывода в консоль: none/noxfer/progress |
|
|
||||||
| `size=N` | Задать размер выходного файла/образа (по умолчанию равен размеру входа).|
|
|
||||||
| `iflag=direct` | Только для входного файла: использовать прямой ввод-вывод |
|
|
||||||
| `oflag=direct` | Только для выходного файла: использовать прямой ввод-вывод |
|
|
||||||
| `oflag=append` | Только для файлов: дописывать в конец выходного файла |
|
|
||||||
| `conv=nocreat` | Не создавать выходной файл/образ |
|
|
||||||
| `conv=trunc` | Обрезать выходной файл/образ до размера входа |
|
|
||||||
| `conv=noerror` | Продолжать копирование после ошибок |
|
|
||||||
| `conv=nofsync` | Не вызывать fsync перед завершением |
|
|
||||||
| `conv=nosparse` | Записывать все выходные блоки, включая пустые |
|
|
||||||
|
|
||||||
## rm
|
## rm
|
||||||
|
|
||||||
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
||||||
|
|
||||||
`vitastor-cli rm (--exact|--matching) <glob> ...`
|
Удалить образ `<from>` или все слои от `<from>` до `<to>` (`<to>` должен быть дочерним
|
||||||
|
образом `<from>`), одновременно меняя родительские образы их клонов (если таковые есть).
|
||||||
|
|
||||||
Удалить образ(ы), корректно перебазируя их дочерние образы.
|
`--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
|
||||||
|
у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
|
||||||
|
В этом случае дочерний образ вливается в родительский и удаляется, а родительский
|
||||||
|
переименовывается в дочерний.
|
||||||
|
|
||||||
В первой форме удаляет один образ `<from>` или все слои между `<from>` и его дочерним `<to>`.
|
В других случаях родительские слои вливаются в дочерние.
|
||||||
|
|
||||||
Во второй форме, удаляет все образы с точными именами или именами, подходящими под шаблон(ы).
|
Другие опции:
|
||||||
|
|
||||||
Опции:
|
* `--down-ok` - Продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
|
||||||
|
|
||||||
* `--writers-stopped` позволяет чуть более эффективно удалять образы в частом случае, когда
|
|
||||||
у удаляемой цепочки есть только один дочерний образ, содержащий небольшой объём данных.
|
|
||||||
В этом случае дочерний образ вливается в родительский и удаляется, а родительский
|
|
||||||
переименовывается в дочерний.
|
|
||||||
* `--exact` - удалить все образы с именами, подходящими под переданные glob-шаблоны.
|
|
||||||
* `--matching` - удалить все образы с точно заданными именами.
|
|
||||||
* `--down-ok` - продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
|
|
||||||
|
|
||||||
## flatten
|
## flatten
|
||||||
|
|
||||||
|
@ -225,7 +182,6 @@ vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
|
||||||
--wait-list Сначала запросить полный листинг объектов, а потом начать удалять.
|
--wait-list Сначала запросить полный листинг объектов, а потом начать удалять.
|
||||||
Требует больше памяти, но позволяет правильно печатать прогресс удаления.
|
Требует больше памяти, но позволяет правильно печатать прогресс удаления.
|
||||||
--min-offset Удалять только данные, начиная с заданного смещения.
|
--min-offset Удалять только данные, начиная с заданного смещения.
|
||||||
--max-offset Удалять только данные до (исключительно) заданного смещения.
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## merge-data
|
## merge-data
|
||||||
|
@ -307,83 +263,6 @@ vitastor-cli dd [iimg=<image> | if=<file>] [oimg=<image> | of=<file>] [bs=1M] \
|
||||||
С опцией `--dry-run` только проверяет, возможно ли удаление без потери данных и деградации
|
С опцией `--dry-run` только проверяет, возможно ли удаление без потери данных и деградации
|
||||||
избыточности.
|
избыточности.
|
||||||
|
|
||||||
## osd-tree
|
|
||||||
|
|
||||||
`vitastor-cli osd-tree [-l|--long]`
|
|
||||||
|
|
||||||
Показать дерево OSD, со статистикой ввода-вывода, если установлено -l.
|
|
||||||
|
|
||||||
Пример вывода:
|
|
||||||
|
|
||||||
```
|
|
||||||
TYPE NAME UP SIZE USED% TAGS WEIGHT BLOCK BITMAP IMM NOOUT
|
|
||||||
host kaveri
|
|
||||||
disk nvme0n1p1
|
|
||||||
osd 3 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
osd 4 down 100G 0 % 1 128k 4k none -
|
|
||||||
disk nvme1n1p1
|
|
||||||
osd 5 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
osd 6 down 100G 0 % 1 128k 4k none -
|
|
||||||
host stump
|
|
||||||
osd 1 up 100G 37.29 % osdone 1 128k 4k all -
|
|
||||||
osd 2 up 100G 26.8 % globl 1 128k 4k all -
|
|
||||||
osd 7 up 100G 21.84 % 1 128k 4k all -
|
|
||||||
osd 8 up 100G 21.63 % 1 128k 4k all -
|
|
||||||
osd 9 up 100G 20.69 % 1 128k 4k all -
|
|
||||||
osd 10 up 100G 21.61 % 1 128k 4k all -
|
|
||||||
osd 11 up 100G 21.53 % 1 128k 4k all -
|
|
||||||
osd 12 up 100G 22.4 % 1 128k 4k all -
|
|
||||||
```
|
|
||||||
|
|
||||||
## ls-osd
|
|
||||||
|
|
||||||
`vitastor-cli osds|ls-osd|osd-ls [-l|--long]`
|
|
||||||
|
|
||||||
Показать список OSD, со статистикой ввода-вывода, если установлено -l.
|
|
||||||
|
|
||||||
Пример вывода:
|
|
||||||
|
|
||||||
```
|
|
||||||
OSD PARENT UP SIZE USED% TAGS WEIGHT BLOCK BITMAP IMM NOOUT
|
|
||||||
3 kaveri/nvme0n1p1 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
4 kaveri/nvme0n1p1 down 100G 0 % 1 128k 4k none -
|
|
||||||
5 kaveri/nvme1n1p1 down 100G 0 % globl,kaveri 1 128k 4k none -
|
|
||||||
6 kaveri/nvme1n1p1 down 100G 0 % 1 128k 4k none -
|
|
||||||
1 stump up 100G 37.29 % osdone 1 128k 4k all -
|
|
||||||
2 stump up 100G 26.8 % globl 1 128k 4k all -
|
|
||||||
7 stump up 100G 21.84 % 1 128k 4k all -
|
|
||||||
8 stump up 100G 21.63 % 1 128k 4k all -
|
|
||||||
9 stump up 100G 20.69 % 1 128k 4k all -
|
|
||||||
10 stump up 100G 21.61 % 1 128k 4k all -
|
|
||||||
11 stump up 100G 21.53 % 1 128k 4k all -
|
|
||||||
12 stump up 100G 22.4 % 1 128k 4k all -
|
|
||||||
```
|
|
||||||
|
|
||||||
## modify-osd
|
|
||||||
|
|
||||||
`vitastor-cli modify-osd [--tags tag1,tag2,...] [--reweight <number>] [--noout true/false] <osd_number>`
|
|
||||||
|
|
||||||
Установить вес OSD, теги или флаг noout. Смотрите подробное описание в [документации настроек OSD](../config/pool.ru.md#настройки-osd).
|
|
||||||
|
|
||||||
## pg-list
|
|
||||||
|
|
||||||
`vitastor-cli pg-list|pg-ls|list-pg|ls-pg|ls-pgs [OPTIONS] [state1+state2] [^state3] [...]`
|
|
||||||
|
|
||||||
Вывести список PG с состояними, удовлетворяющими любому из переданных фильтров (^ или !
|
|
||||||
в начале фильтра означает отрицание). Опции:
|
|
||||||
|
|
||||||
```
|
|
||||||
--pool <pool name or number> Only list PGs of the given pool.
|
|
||||||
--min <min pg number> Only list PGs with number >= min.
|
|
||||||
--max <max pg number> Only list PGs with number <= max.
|
|
||||||
```
|
|
||||||
|
|
||||||
Примеры:
|
|
||||||
|
|
||||||
`vitastor-cli pg-list active+degraded`
|
|
||||||
|
|
||||||
`vitastor-cli pg-list ^active`
|
|
||||||
|
|
||||||
## create-pool
|
## create-pool
|
||||||
|
|
||||||
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
||||||
|
|
|
@ -11,8 +11,6 @@ Vitastor has two file system implementations. Both can be used via `vitastor-nfs
|
||||||
Commands:
|
Commands:
|
||||||
- [mount](#mount)
|
- [mount](#mount)
|
||||||
- [start](#start)
|
- [start](#start)
|
||||||
- [upgrade](#upgrade)
|
|
||||||
- [defrag](#defrag)
|
|
||||||
|
|
||||||
## Pseudo-FS
|
## Pseudo-FS
|
||||||
|
|
||||||
|
@ -88,6 +86,10 @@ POSIX features currently not implemented in VitastorFS:
|
||||||
- Modification time (`mtime`) is updated lazily every second (like `-o lazytime`)
|
- Modification time (`mtime`) is updated lazily every second (like `-o lazytime`)
|
||||||
|
|
||||||
Other notable missing features which should be addressed in the future:
|
Other notable missing features which should be addressed in the future:
|
||||||
|
- Defragmentation of "shared" inodes. Files smaller than pool object size (block_size
|
||||||
|
multiplied by data part count if pool is EC) are internally stored in large block
|
||||||
|
volumes sequentially, one after another, and leave garbage after deleting or resizing.
|
||||||
|
Defragmentator will be implemented to collect this garbage.
|
||||||
- Inode ID reuse. Currently inode IDs always grow, the limit is 2^48 inodes, so
|
- Inode ID reuse. Currently inode IDs always grow, the limit is 2^48 inodes, so
|
||||||
in theory you may hit it if you create and delete a very large number of files
|
in theory you may hit it if you create and delete a very large number of files
|
||||||
- Compaction of the key-value B-Tree. Current implementation never merges or deletes
|
- Compaction of the key-value B-Tree. Current implementation never merges or deletes
|
||||||
|
@ -137,37 +139,6 @@ Start network NFS server. Options:
|
||||||
| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049) |
|
| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049) |
|
||||||
| `--portmap 0` | do not listen on port 111 (portmap/rpcbind, requires root) |
|
| `--portmap 0` | do not listen on port 111 (portmap/rpcbind, requires root) |
|
||||||
|
|
||||||
### upgrade
|
|
||||||
|
|
||||||
`vitastor-nfs --fs <NAME> upgrade`
|
|
||||||
|
|
||||||
Upgrade FS metadata. Can be run online, but server(s) should be restarted after upgrade.
|
|
||||||
|
|
||||||
### defrag
|
|
||||||
|
|
||||||
`vitastor-nfs --fs <NAME> defrag [OPTIONS] [--dry-run]`
|
|
||||||
|
|
||||||
Defragment volumes used for small file storage having more than \<defrag_percent> %
|
|
||||||
of data removed. Can be run online.
|
|
||||||
|
|
||||||
In VitastorFS, small files are stored in large "volumes" / "shared inodes" one
|
|
||||||
after another. When you delete or extend such files, they are moved and garbage is left
|
|
||||||
behind. Defragmentation removes garbage and moves data still in use to new volumes.
|
|
||||||
|
|
||||||
Options:
|
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
|
||||||
|--------------------------|------------------------------------------------------------------------ |
|
|
||||||
| --volume_untouched 86400 | Defragment volumes last appended to at least this number of seconds ago |
|
|
||||||
| --defrag_percent 50 | Defragment volumes with at least this % of removed data |
|
|
||||||
| --defrag_block_count 16 | Read this number of pool blocks at once during defrag |
|
|
||||||
| --defrag_iodepth 16 | Move up to this number of files in parallel during defrag |
|
|
||||||
| --trace | Print verbose defragmentation status |
|
|
||||||
| --dry-run | Skip modifications, only print status |
|
|
||||||
| --recalc-stats | Recalculate all volume statistics |
|
|
||||||
| --include-empty | Include old and empty volumes; make sure to restart NFS servers before using it |
|
|
||||||
| --no-rm | Move, but do not delete data |
|
|
||||||
|
|
||||||
## Common options
|
## Common options
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
| <!-- --> | <!-- --> |
|
||||||
|
|
|
@ -11,8 +11,6 @@
|
||||||
Команды:
|
Команды:
|
||||||
- [mount](#mount)
|
- [mount](#mount)
|
||||||
- [start](#start)
|
- [start](#start)
|
||||||
- [upgrade](#upgrade)
|
|
||||||
- [defrag](#defrag)
|
|
||||||
|
|
||||||
## Псевдо-ФС
|
## Псевдо-ФС
|
||||||
|
|
||||||
|
@ -90,6 +88,11 @@ JSON-формате :-). Для инспекции содержимого БД
|
||||||
- Времена модификации (`mtime`) отслеживаются асинхронно (как будто ФС смонтирована с `-o lazytime`)
|
- Времена модификации (`mtime`) отслеживаются асинхронно (как будто ФС смонтирована с `-o lazytime`)
|
||||||
|
|
||||||
Другие недостающие функции, которые нужно добавить в будущем:
|
Другие недостающие функции, которые нужно добавить в будущем:
|
||||||
|
- Дефрагментация "общих инодов". На уровне реализации ФС файлы, меньшие, чем размер
|
||||||
|
объекта пула (block_size умножить на число частей данных, если пул EC),
|
||||||
|
упаковываются друг за другом в большие "общие" иноды/тома. Если такие файлы удалять
|
||||||
|
или увеличивать, они перемещаются и оставляют за собой "мусор", вот тут-то и нужен
|
||||||
|
дефрагментатор.
|
||||||
- Переиспользование номеров инодов. В текущей реализации номера инодов всё время
|
- Переиспользование номеров инодов. В текущей реализации номера инодов всё время
|
||||||
увеличиваются, так что в теории вы можете упереться в лимит, если насоздаёте
|
увеличиваются, так что в теории вы можете упереться в лимит, если насоздаёте
|
||||||
и наудаляете больше, чем 2^48 файлов.
|
и наудаляете больше, чем 2^48 файлов.
|
||||||
|
@ -142,40 +145,6 @@ JSON-формате :-). Для инспекции содержимого БД
|
||||||
| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049) |
|
| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049) |
|
||||||
| `--portmap 0` | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
|
| `--portmap 0` | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
|
||||||
|
|
||||||
### upgrade
|
|
||||||
|
|
||||||
`vitastor-nfs --fs <NAME> upgrade`
|
|
||||||
|
|
||||||
Обновить метаданные ФС. Можно запускать онлайн (при запущенных серверах NFS), но после выполнения их всё
|
|
||||||
же желательно перезапустить.
|
|
||||||
|
|
||||||
### defrag
|
|
||||||
|
|
||||||
`vitastor-nfs --fs <NAME> defrag [OPTIONS] [--dry-run]`
|
|
||||||
|
|
||||||
Дефрагментировать тома, используемые для хранения мелких файлов, в которых более, чем
|
|
||||||
<defrag_percent> процентов данных удалено. Можно запускать онлайн.
|
|
||||||
|
|
||||||
На уровне реализации ФС файлы, меньшие, чем размер объекта пула (block_size умножить на число
|
|
||||||
частей данных, если пул EC), упаковываются друг за другом в большие "тома" / "общие иноды".
|
|
||||||
Когда такие файлы удаляются или увеличиваются, они перемещаются и оставляют за собой "мусор".
|
|
||||||
|
|
||||||
При дефрагментации мусор удаляется, а всё ещё используемые данные перемещаются в новые тома.
|
|
||||||
|
|
||||||
Опции:
|
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
|
||||||
|--------------------------|------------------------------------------------------------------------ |
|
|
||||||
| --volume_untouched 86400 | Дефрагментировать только тома, в которые уже не писали это число секунд |
|
|
||||||
| --defrag_percent 50 | Дефрагментировать только тома, в которых этот % данных удалён |
|
|
||||||
| --defrag_block_count 16 | Читать это количество блоков пула за один раз |
|
|
||||||
| --defrag_iodepth 16 | Перемещать одновременно до этого числа файлов |
|
|
||||||
| --trace | Печатать детальную статистику дефрагментации |
|
|
||||||
| --dry-run | Не производить никаких изменений, только описать выполняемые действия |
|
|
||||||
| --recalc-stats | Пересчитать и сохранить статистику всех томов |
|
|
||||||
| --include-empty | Дефрагментировать старые и пустые тома; обязательно перезапустите NFS-сервера после использования этой опции |
|
|
||||||
| --no-rm | Перемещать, но не удалять данные |
|
|
||||||
|
|
||||||
## Общие опции
|
## Общие опции
|
||||||
|
|
||||||
| <!-- --> | <!-- --> |
|
| <!-- --> | <!-- --> |
|
||||||
|
|
|
@ -11,7 +11,6 @@ module.exports = {
|
||||||
"ecmaVersion": 2020
|
"ecmaVersion": 2020
|
||||||
},
|
},
|
||||||
"plugins": [
|
"plugins": [
|
||||||
"import"
|
|
||||||
],
|
],
|
||||||
"rules": {
|
"rules": {
|
||||||
"indent": [
|
"indent": [
|
||||||
|
@ -45,10 +44,6 @@ module.exports = {
|
||||||
],
|
],
|
||||||
"node/shebang": [
|
"node/shebang": [
|
||||||
"off"
|
"off"
|
||||||
],
|
|
||||||
"import/no-unresolved": [
|
|
||||||
2,
|
|
||||||
{ "commonjs": true }
|
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
|
@ -1,188 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const AntiEtcd = require('antietcd');
|
|
||||||
|
|
||||||
const vitastor_persist_filter = require('./vitastor_persist_filter.js');
|
|
||||||
const { b64, local_ips } = require('./utils.js');
|
|
||||||
|
|
||||||
class AntiEtcdAdapter
|
|
||||||
{
|
|
||||||
static async start_antietcd(config)
|
|
||||||
{
|
|
||||||
let antietcd;
|
|
||||||
if (config.use_antietcd)
|
|
||||||
{
|
|
||||||
let cluster = config.etcd_address;
|
|
||||||
if (!(cluster instanceof Array))
|
|
||||||
cluster = cluster ? (''+(cluster||'')).split(/,+/) : [];
|
|
||||||
cluster = Object.keys(cluster.reduce((a, url) =>
|
|
||||||
{
|
|
||||||
a[url.toLowerCase().replace(/^(https?:\/\/)/, '').replace(/\/.*$/, '')] = true;
|
|
||||||
return a;
|
|
||||||
}, {}));
|
|
||||||
const cfg_port = config.antietcd_port;
|
|
||||||
const is_local = local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
|
|
||||||
const selected = cluster.map(s => s.split(':', 2)).filter(ip => is_local[ip[0]] && (!cfg_port || ip[1] == cfg_port));
|
|
||||||
if (selected.length > 1)
|
|
||||||
{
|
|
||||||
console.error('More than 1 etcd_address matches local IPs, please specify port');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
else if (selected.length == 1)
|
|
||||||
{
|
|
||||||
const antietcd_config = {
|
|
||||||
ip: selected[0][0],
|
|
||||||
port: selected[0][1],
|
|
||||||
data: config.antietcd_data_file || ((config.antietcd_data_dir || '/var/lib/vitastor') + '/mon_'+selected[0][1]+'.json.gz'),
|
|
||||||
persist_filter: vitastor_persist_filter({ vitastor_prefix: config.etcd_prefix || '/vitastor' }),
|
|
||||||
node_id: selected[0][0]+':'+selected[0][1], // node_id = ip:port
|
|
||||||
cluster: (cluster.length == 1 ? null : cluster.reduce((a, c) => { a[c] = "http://"+c; return a; }, {})),
|
|
||||||
cluster_key: (config.etcd_prefix || '/vitastor'),
|
|
||||||
stale_read: 1,
|
|
||||||
log_level: 1,
|
|
||||||
};
|
|
||||||
for (const key in config)
|
|
||||||
{
|
|
||||||
if (key.substr(0, 9) === 'antietcd_')
|
|
||||||
{
|
|
||||||
const noprefix = key.substr(9);
|
|
||||||
if (!(noprefix in antietcd_config) || noprefix == 'ip' || noprefix == 'cluster_key')
|
|
||||||
{
|
|
||||||
antietcd_config[noprefix] = config[key];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
console.log('Starting Antietcd node '+antietcd_config.node_id);
|
|
||||||
antietcd = new AntiEtcd(antietcd_config);
|
|
||||||
await antietcd.start();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
console.log('Antietcd is enabled, but etcd_address does not contain local IPs, proceeding without it');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return antietcd;
|
|
||||||
}
|
|
||||||
|
|
||||||
constructor(mon, antietcd)
|
|
||||||
{
|
|
||||||
this.mon = mon;
|
|
||||||
this.antietcd = antietcd;
|
|
||||||
this.on_leader = [];
|
|
||||||
this.on_change = (st) =>
|
|
||||||
{
|
|
||||||
if (st.state === 'leader')
|
|
||||||
{
|
|
||||||
for (const cb of this.on_leader)
|
|
||||||
{
|
|
||||||
cb();
|
|
||||||
}
|
|
||||||
this.on_leader = [];
|
|
||||||
}
|
|
||||||
};
|
|
||||||
this.antietcd.on('raftchange', this.on_change);
|
|
||||||
}
|
|
||||||
|
|
||||||
parse_config(/*config*/)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_watcher()
|
|
||||||
{
|
|
||||||
this.antietcd.off('raftchange', this.on_change);
|
|
||||||
const watch_id = this.watch_id;
|
|
||||||
if (watch_id)
|
|
||||||
{
|
|
||||||
this.watch_id = null;
|
|
||||||
this.antietcd.cancel_watch(watch_id).catch(console.error);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async start_watcher()
|
|
||||||
{
|
|
||||||
if (this.watch_id)
|
|
||||||
{
|
|
||||||
await this.antietcd.cancel_watch(this.watch_id);
|
|
||||||
this.watch_id = null;
|
|
||||||
}
|
|
||||||
const watch_id = await this.antietcd.create_watch({
|
|
||||||
key: b64(this.mon.config.etcd_prefix+'/'),
|
|
||||||
range_end: b64(this.mon.config.etcd_prefix+'0'),
|
|
||||||
start_revision: ''+this.mon.etcd_watch_revision,
|
|
||||||
watch_id: 1,
|
|
||||||
progress_notify: true,
|
|
||||||
}, (message) =>
|
|
||||||
{
|
|
||||||
setImmediate(() => this.mon.on_message(message.result));
|
|
||||||
});
|
|
||||||
console.log('Successfully subscribed to antietcd revision '+this.antietcd.etctree.mod_revision);
|
|
||||||
this.watch_id = watch_id;
|
|
||||||
}
|
|
||||||
|
|
||||||
async become_master()
|
|
||||||
{
|
|
||||||
if (!this.antietcd.cluster)
|
|
||||||
{
|
|
||||||
console.log('Running in non-clustered mode');
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
console.log('Waiting to become master');
|
|
||||||
if (this.antietcd.cluster.raft.state !== 'leader')
|
|
||||||
{
|
|
||||||
await new Promise(ok => this.on_leader.push(ok));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const state = { ...this.mon.get_mon_state(), id: ''+this.mon.etcd_lease_id };
|
|
||||||
await this.etcd_call('/kv/txn', {
|
|
||||||
success: [ { requestPut: { key: b64(this.mon.config.etcd_prefix+'/mon/master'), value: b64(JSON.stringify(state)), lease: ''+this.mon.etcd_lease_id } } ],
|
|
||||||
}, this.mon.config.etcd_start_timeout, 0);
|
|
||||||
if (this.antietcd.cluster)
|
|
||||||
{
|
|
||||||
console.log('Became master');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async etcd_call(path, body, timeout, retries)
|
|
||||||
{
|
|
||||||
let retry = 0;
|
|
||||||
if (retries >= 0 && retries < 1)
|
|
||||||
{
|
|
||||||
retries = 1;
|
|
||||||
}
|
|
||||||
let prev = 0;
|
|
||||||
while (retries < 0 || retry < retries)
|
|
||||||
{
|
|
||||||
retry++;
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
throw new Error('Monitor instance is stopped');
|
|
||||||
}
|
|
||||||
try
|
|
||||||
{
|
|
||||||
if (Date.now()-prev < timeout)
|
|
||||||
{
|
|
||||||
await new Promise(ok => setTimeout(ok, timeout-(Date.now()-prev)));
|
|
||||||
}
|
|
||||||
prev = Date.now();
|
|
||||||
const res = await this.antietcd.api(path.replace(/^\/+/, '').replace(/\/+$/, '').replace(/\/+/g, '_'), body);
|
|
||||||
if (res.error)
|
|
||||||
{
|
|
||||||
console.error('Failed to query antietcd '+path+' (retry '+retry+'/'+retries+'): '+res.error);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
console.error('Failed to query antietcd '+path+' (retry '+retry+'/'+retries+'): '+e.stack);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Failed to query antietcd ('+retries+' retries)');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = AntiEtcdAdapter;
|
|
|
@ -1,352 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const http = require('http');
|
|
||||||
const WebSocket = require('ws');
|
|
||||||
const { b64, local_ips } = require('./utils.js');
|
|
||||||
|
|
||||||
const MON_STOPPED = 'Monitor instance is stopped';
|
|
||||||
|
|
||||||
class EtcdAdapter
|
|
||||||
{
|
|
||||||
constructor(mon)
|
|
||||||
{
|
|
||||||
this.mon = mon;
|
|
||||||
this.ws = null;
|
|
||||||
this.ws_alive = false;
|
|
||||||
this.ws_keepalive_timer = null;
|
|
||||||
}
|
|
||||||
|
|
||||||
parse_config(config)
|
|
||||||
{
|
|
||||||
this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
|
|
||||||
}
|
|
||||||
|
|
||||||
parse_etcd_addresses(addrs)
|
|
||||||
{
|
|
||||||
const is_local_ip = local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
|
|
||||||
this.etcd_local = [];
|
|
||||||
this.etcd_urls = [];
|
|
||||||
this.selected_etcd_url = null;
|
|
||||||
this.etcd_urls_to_try = [];
|
|
||||||
if (!(addrs instanceof Array))
|
|
||||||
addrs = addrs ? (''+(addrs||'')).split(/,/) : [];
|
|
||||||
if (!addrs.length)
|
|
||||||
{
|
|
||||||
console.error('Vitastor etcd address(es) not specified. Please set on the command line or in the config file');
|
|
||||||
process.exit(1);
|
|
||||||
}
|
|
||||||
for (let url of addrs)
|
|
||||||
{
|
|
||||||
let scheme = 'http';
|
|
||||||
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
|
|
||||||
const slash = url.indexOf('/');
|
|
||||||
const colon = url.indexOf(':');
|
|
||||||
const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
|
|
||||||
url = scheme+'://'+(slash >= 0 ? url : url+'/v3');
|
|
||||||
if (is_local)
|
|
||||||
this.etcd_local.push(url);
|
|
||||||
else
|
|
||||||
this.etcd_urls.push(url);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pick_next_etcd()
|
|
||||||
{
|
|
||||||
if (this.selected_etcd_url)
|
|
||||||
return this.selected_etcd_url;
|
|
||||||
if (!this.etcd_urls_to_try || !this.etcd_urls_to_try.length)
|
|
||||||
{
|
|
||||||
this.etcd_urls_to_try = [ ...this.etcd_local ];
|
|
||||||
const others = [ ...this.etcd_urls ];
|
|
||||||
while (others.length)
|
|
||||||
{
|
|
||||||
const url = others.splice(0|(others.length*Math.random()), 1);
|
|
||||||
this.etcd_urls_to_try.push(url[0]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
this.selected_etcd_url = this.etcd_urls_to_try.shift();
|
|
||||||
return this.selected_etcd_url;
|
|
||||||
}
|
|
||||||
|
|
||||||
stop_watcher(cur_addr)
|
|
||||||
{
|
|
||||||
cur_addr = cur_addr || this.selected_etcd_url;
|
|
||||||
if (this.ws)
|
|
||||||
{
|
|
||||||
console.log('Disconnected from etcd at '+this.ws_used_url);
|
|
||||||
this.ws.close();
|
|
||||||
this.ws = null;
|
|
||||||
}
|
|
||||||
if (this.ws_keepalive_timer)
|
|
||||||
{
|
|
||||||
clearInterval(this.ws_keepalive_timer);
|
|
||||||
this.ws_keepalive_timer = null;
|
|
||||||
}
|
|
||||||
if (this.selected_etcd_url == cur_addr)
|
|
||||||
{
|
|
||||||
this.selected_etcd_url = null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
restart_watcher(cur_addr)
|
|
||||||
{
|
|
||||||
this.stop_watcher(cur_addr);
|
|
||||||
this.start_watcher(this.mon.config.etcd_mon_retries).catch(this.mon.die);
|
|
||||||
}
|
|
||||||
|
|
||||||
async start_watcher(retries)
|
|
||||||
{
|
|
||||||
let retry = 0;
|
|
||||||
if (!retries || retries < 1)
|
|
||||||
{
|
|
||||||
retries = 1;
|
|
||||||
}
|
|
||||||
const tried = {};
|
|
||||||
while (retries < 0 || retry < retries)
|
|
||||||
{
|
|
||||||
const cur_addr = this.pick_next_etcd();
|
|
||||||
const base = 'ws'+cur_addr.substr(4);
|
|
||||||
let now = Date.now();
|
|
||||||
if (tried[base] && now-tried[base] < this.mon.config.etcd_start_timeout)
|
|
||||||
{
|
|
||||||
await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout-(now-tried[base])));
|
|
||||||
now = Date.now();
|
|
||||||
}
|
|
||||||
tried[base] = now;
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const ok = await new Promise(ok =>
|
|
||||||
{
|
|
||||||
const timer_id = setTimeout(() =>
|
|
||||||
{
|
|
||||||
if (this.ws)
|
|
||||||
{
|
|
||||||
console.log('Disconnected from etcd at '+this.ws_used_url);
|
|
||||||
this.ws.close();
|
|
||||||
this.ws = null;
|
|
||||||
}
|
|
||||||
ok(false);
|
|
||||||
}, this.mon.config.etcd_mon_timeout);
|
|
||||||
this.ws = new WebSocket(base+'/watch');
|
|
||||||
this.ws_used_url = cur_addr;
|
|
||||||
const fail = () =>
|
|
||||||
{
|
|
||||||
ok(false);
|
|
||||||
};
|
|
||||||
this.ws.on('error', fail);
|
|
||||||
this.ws.on('open', () =>
|
|
||||||
{
|
|
||||||
this.ws.removeListener('error', fail);
|
|
||||||
if (timer_id)
|
|
||||||
clearTimeout(timer_id);
|
|
||||||
ok(true);
|
|
||||||
});
|
|
||||||
});
|
|
||||||
if (ok)
|
|
||||||
break;
|
|
||||||
if (this.selected_etcd_url == cur_addr)
|
|
||||||
this.selected_etcd_url = null;
|
|
||||||
this.ws = null;
|
|
||||||
retry++;
|
|
||||||
}
|
|
||||||
if (!this.ws)
|
|
||||||
{
|
|
||||||
this.mon.die('Failed to open etcd watch websocket');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
this.stop_watcher();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const cur_addr = this.selected_etcd_url;
|
|
||||||
this.ws_alive = true;
|
|
||||||
this.ws_keepalive_timer = setInterval(() =>
|
|
||||||
{
|
|
||||||
if (this.ws_alive && this.ws)
|
|
||||||
{
|
|
||||||
this.ws_alive = false;
|
|
||||||
this.ws.send(JSON.stringify({ progress_request: {} }));
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
console.log('etcd websocket timed out, restarting it');
|
|
||||||
this.restart_watcher(cur_addr);
|
|
||||||
}
|
|
||||||
}, (Number(this.mon.config.etcd_ws_keepalive_interval) || 5)*1000);
|
|
||||||
this.ws.on('error', () => this.restart_watcher(cur_addr));
|
|
||||||
this.ws.send(JSON.stringify({
|
|
||||||
create_request: {
|
|
||||||
key: b64(this.mon.config.etcd_prefix+'/'),
|
|
||||||
range_end: b64(this.mon.config.etcd_prefix+'0'),
|
|
||||||
start_revision: ''+this.mon.etcd_watch_revision,
|
|
||||||
watch_id: 1,
|
|
||||||
progress_notify: true,
|
|
||||||
},
|
|
||||||
}));
|
|
||||||
this.ws.on('message', (msg) =>
|
|
||||||
{
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
this.stop_watcher();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
this.ws_alive = true;
|
|
||||||
let data;
|
|
||||||
try
|
|
||||||
{
|
|
||||||
data = JSON.parse(msg);
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
if (!data || !data.result)
|
|
||||||
{
|
|
||||||
console.error('Unknown message received from watch websocket: '+msg);
|
|
||||||
}
|
|
||||||
else if (data.result.canceled)
|
|
||||||
{
|
|
||||||
// etcd watch canceled
|
|
||||||
if (data.result.compact_revision)
|
|
||||||
{
|
|
||||||
// we may miss events if we proceed
|
|
||||||
this.mon.die('Revisions before '+data.result.compact_revision+' were compacted by etcd, exiting');
|
|
||||||
}
|
|
||||||
this.mon.die('Watch canceled by etcd, reason: '+data.result.cancel_reason+', exiting');
|
|
||||||
}
|
|
||||||
else if (data.result.created)
|
|
||||||
{
|
|
||||||
// etcd watch created
|
|
||||||
console.log('Successfully subscribed to etcd at '+this.selected_etcd_url+', revision '+data.result.header.revision);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
this.mon.on_message(data.result);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
async become_master()
|
|
||||||
{
|
|
||||||
const state = { ...this.mon.get_mon_state(), id: ''+this.mon.etcd_lease_id };
|
|
||||||
// eslint-disable-next-line no-constant-condition
|
|
||||||
while (1)
|
|
||||||
{
|
|
||||||
const res = await this.etcd_call('/kv/txn', {
|
|
||||||
compare: [ { target: 'CREATE', create_revision: 0, key: b64(this.mon.config.etcd_prefix+'/mon/master') } ],
|
|
||||||
success: [ { requestPut: { key: b64(this.mon.config.etcd_prefix+'/mon/master'), value: b64(JSON.stringify(state)), lease: ''+this.mon.etcd_lease_id } } ],
|
|
||||||
}, this.mon.config.etcd_start_timeout, 0);
|
|
||||||
if (res.succeeded)
|
|
||||||
{
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
console.log('Waiting to become master');
|
|
||||||
await new Promise(ok => setTimeout(ok, this.mon.config.etcd_start_timeout));
|
|
||||||
}
|
|
||||||
console.log('Became master');
|
|
||||||
}
|
|
||||||
|
|
||||||
async etcd_call(path, body, timeout, retries)
|
|
||||||
{
|
|
||||||
let retry = 0;
|
|
||||||
if (retries >= 0 && retries < 1)
|
|
||||||
{
|
|
||||||
retries = 1;
|
|
||||||
}
|
|
||||||
const tried = {};
|
|
||||||
while (retries < 0 || retry < retries)
|
|
||||||
{
|
|
||||||
retry++;
|
|
||||||
const base = this.pick_next_etcd();
|
|
||||||
let now = Date.now();
|
|
||||||
if (tried[base] && now-tried[base] < timeout)
|
|
||||||
{
|
|
||||||
await new Promise(ok => setTimeout(ok, timeout-(now-tried[base])));
|
|
||||||
now = Date.now();
|
|
||||||
}
|
|
||||||
tried[base] = now;
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
throw new Error(MON_STOPPED);
|
|
||||||
}
|
|
||||||
const res = await POST(base+path, body, timeout);
|
|
||||||
if (this.mon.stopped)
|
|
||||||
{
|
|
||||||
throw new Error(MON_STOPPED);
|
|
||||||
}
|
|
||||||
if (res.error)
|
|
||||||
{
|
|
||||||
if (this.selected_etcd_url == base)
|
|
||||||
this.selected_etcd_url = null;
|
|
||||||
console.error('Failed to query etcd '+path+' (retry '+retry+'/'+retries+'): '+res.error);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (res.json)
|
|
||||||
{
|
|
||||||
if (res.json.error)
|
|
||||||
{
|
|
||||||
console.error(path+': etcd returned error: '+res.json.error);
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
return res.json;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
throw new Error('Failed to query etcd ('+retries+' retries)');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function POST(url, body, timeout)
|
|
||||||
{
|
|
||||||
return new Promise(ok =>
|
|
||||||
{
|
|
||||||
const body_text = Buffer.from(JSON.stringify(body));
|
|
||||||
let timer_id = timeout > 0 ? setTimeout(() =>
|
|
||||||
{
|
|
||||||
if (req)
|
|
||||||
req.abort();
|
|
||||||
req = null;
|
|
||||||
ok({ error: 'timeout' });
|
|
||||||
}, timeout) : null;
|
|
||||||
let req = http.request(url, { method: 'POST', headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
'Content-Length': body_text.length,
|
|
||||||
} }, (res) =>
|
|
||||||
{
|
|
||||||
if (!req)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
clearTimeout(timer_id);
|
|
||||||
let res_body = '';
|
|
||||||
res.setEncoding('utf8');
|
|
||||||
res.on('error', (error) => ok({ error }));
|
|
||||||
res.on('data', chunk => { res_body += chunk; });
|
|
||||||
res.on('end', () =>
|
|
||||||
{
|
|
||||||
if (res.statusCode != 200)
|
|
||||||
{
|
|
||||||
ok({ error: res_body, code: res.statusCode });
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
try
|
|
||||||
{
|
|
||||||
res_body = JSON.parse(res_body);
|
|
||||||
ok({ response: res, json: res_body });
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
ok({ error: e, response: res, body: res_body });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
});
|
|
||||||
req.on('error', (error) => ok({ error }));
|
|
||||||
req.on('close', () => ok({ error: new Error('Connection closed prematurely') }));
|
|
||||||
req.write(body_text);
|
|
||||||
req.end();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = EtcdAdapter;
|
|
|
@ -1,396 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
// FIXME document all etcd keys and config variables in the form of JSON schema or similar
|
|
||||||
const etcd_nonempty_keys = {
|
|
||||||
'config/global': 1,
|
|
||||||
'config/node_placement': 1,
|
|
||||||
'config/pools': 1,
|
|
||||||
'pg/config': 1,
|
|
||||||
'history/last_clean_pgs': 1,
|
|
||||||
'stats': 1,
|
|
||||||
};
|
|
||||||
const etcd_allow = new RegExp('^'+[
|
|
||||||
'config/global',
|
|
||||||
'config/node_placement',
|
|
||||||
'config/pools',
|
|
||||||
'config/osd/[1-9]\\d*',
|
|
||||||
'config/pgs', // old name
|
|
||||||
'pg/config',
|
|
||||||
'config/inode/[1-9]\\d*/[1-9]\\d*',
|
|
||||||
'osd/state/[1-9]\\d*',
|
|
||||||
'osd/stats/[1-9]\\d*',
|
|
||||||
'osd/inodestats/[1-9]\\d*',
|
|
||||||
'osd/space/[1-9]\\d*',
|
|
||||||
'mon/master',
|
|
||||||
'mon/member/[a-f0-9]+',
|
|
||||||
'pg/state/[1-9]\\d*/[1-9]\\d*',
|
|
||||||
'pg/stats/[1-9]\\d*/[1-9]\\d*', // old name
|
|
||||||
'pgstats/[1-9]\\d*/[1-9]\\d*',
|
|
||||||
'pg/history/[1-9]\\d*/[1-9]\\d*',
|
|
||||||
'history/last_clean_pgs',
|
|
||||||
'inode/stats/[1-9]\\d*/\\d+',
|
|
||||||
'pool/stats/[1-9]\\d*',
|
|
||||||
'stats',
|
|
||||||
'index/image/.*',
|
|
||||||
'index/maxid/[1-9]\\d*',
|
|
||||||
].join('$|^')+'$');
|
|
||||||
|
|
||||||
const etcd_tree = {
|
|
||||||
config: {
|
|
||||||
/* global: {
|
|
||||||
// WARNING: NOT ALL OF THESE ARE ACTUALLY CONFIGURABLE HERE
|
|
||||||
// THIS IS JUST A POOR MAN'S CONFIG DOCUMENTATION
|
|
||||||
// etcd connection
|
|
||||||
config_path: "/etc/vitastor/vitastor.conf",
|
|
||||||
etcd_prefix: "/vitastor",
|
|
||||||
// etcd connection - configurable online
|
|
||||||
etcd_address: "10.0.115.10:2379/v3",
|
|
||||||
// mon
|
|
||||||
etcd_mon_ttl: 5, // min: 1
|
|
||||||
etcd_mon_timeout: 1000, // ms. min: 0
|
|
||||||
etcd_mon_retries: 5, // min: 0
|
|
||||||
mon_change_timeout: 1000, // ms. min: 100
|
|
||||||
mon_retry_change_timeout: 50, // ms. min: 10
|
|
||||||
mon_stats_timeout: 1000, // ms. min: 100
|
|
||||||
osd_out_time: 600, // seconds. min: 0
|
|
||||||
placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
|
|
||||||
use_old_pg_combinator: false,
|
|
||||||
// client and osd
|
|
||||||
tcp_header_buffer_size: 65536,
|
|
||||||
use_sync_send_recv: false,
|
|
||||||
use_rdma: true,
|
|
||||||
rdma_device: null, // for example, "rocep5s0f0"
|
|
||||||
rdma_port_num: 1,
|
|
||||||
rdma_gid_index: 0,
|
|
||||||
rdma_mtu: 4096,
|
|
||||||
rdma_max_sge: 128,
|
|
||||||
rdma_max_send: 8,
|
|
||||||
rdma_max_recv: 16,
|
|
||||||
rdma_max_msg: 132096,
|
|
||||||
block_size: 131072,
|
|
||||||
disk_alignment: 4096,
|
|
||||||
bitmap_granularity: 4096,
|
|
||||||
immediate_commit: 'all', // 'none', 'all' or 'small'
|
|
||||||
// client - configurable online
|
|
||||||
client_max_dirty_bytes: 33554432,
|
|
||||||
client_max_dirty_ops: 1024,
|
|
||||||
client_enable_writeback: false,
|
|
||||||
client_max_buffered_bytes: 33554432,
|
|
||||||
client_max_buffered_ops: 1024,
|
|
||||||
client_max_writeback_iodepth: 256,
|
|
||||||
client_retry_interval: 50, // ms. min: 10
|
|
||||||
client_eio_retry_interval: 1000, // ms
|
|
||||||
client_retry_enospc: true,
|
|
||||||
osd_nearfull_ratio: 0.95,
|
|
||||||
// client and osd - configurable online
|
|
||||||
log_level: 0,
|
|
||||||
peer_connect_interval: 5, // seconds. min: 1
|
|
||||||
peer_connect_timeout: 5, // seconds. min: 1
|
|
||||||
osd_idle_timeout: 5, // seconds. min: 1
|
|
||||||
osd_ping_timeout: 5, // seconds. min: 1
|
|
||||||
max_etcd_attempts: 5,
|
|
||||||
etcd_quick_timeout: 1000, // ms
|
|
||||||
etcd_slow_timeout: 5000, // ms
|
|
||||||
etcd_keepalive_timeout: 30, // seconds, default is max(30, etcd_report_interval*2)
|
|
||||||
etcd_ws_keepalive_interval: 5, // seconds
|
|
||||||
// osd
|
|
||||||
etcd_report_interval: 5, // seconds
|
|
||||||
etcd_stats_interval: 30, // seconds
|
|
||||||
run_primary: true,
|
|
||||||
osd_network: null, // "192.168.7.0/24" or an array of masks
|
|
||||||
bind_address: "0.0.0.0",
|
|
||||||
bind_port: 0,
|
|
||||||
readonly: false,
|
|
||||||
osd_memlock: false,
|
|
||||||
// osd - configurable online
|
|
||||||
autosync_interval: 5,
|
|
||||||
autosync_writes: 128,
|
|
||||||
client_queue_depth: 128, // unused
|
|
||||||
recovery_queue_depth: 1,
|
|
||||||
recovery_sleep_us: 0,
|
|
||||||
recovery_tune_util_low: 0.1,
|
|
||||||
recovery_tune_client_util_low: 0,
|
|
||||||
recovery_tune_util_high: 1.0,
|
|
||||||
recovery_tune_client_util_high: 0.5,
|
|
||||||
recovery_tune_interval: 1,
|
|
||||||
recovery_tune_agg_interval: 10, // 10 times recovery_tune_interval
|
|
||||||
recovery_tune_sleep_min_us: 10, // 10 microseconds
|
|
||||||
recovery_pg_switch: 128,
|
|
||||||
recovery_sync_batch: 16,
|
|
||||||
no_recovery: false,
|
|
||||||
no_rebalance: false,
|
|
||||||
print_stats_interval: 3,
|
|
||||||
slow_log_interval: 10,
|
|
||||||
inode_vanish_time: 60,
|
|
||||||
auto_scrub: false,
|
|
||||||
no_scrub: false,
|
|
||||||
scrub_interval: '30d', // 1s/1m/1h/1d
|
|
||||||
scrub_queue_depth: 1,
|
|
||||||
scrub_sleep: 0, // milliseconds
|
|
||||||
scrub_list_limit: 1000, // objects to list on one scrub iteration
|
|
||||||
scrub_find_best: true,
|
|
||||||
scrub_ec_max_bruteforce: 100, // maximum EC error locator brute-force iterators
|
|
||||||
// blockstore - fixed in superblock
|
|
||||||
block_size,
|
|
||||||
disk_alignment,
|
|
||||||
journal_block_size,
|
|
||||||
meta_block_size,
|
|
||||||
bitmap_granularity,
|
|
||||||
journal_device,
|
|
||||||
journal_offset,
|
|
||||||
journal_size,
|
|
||||||
disable_journal_fsync,
|
|
||||||
data_device,
|
|
||||||
data_offset,
|
|
||||||
data_size,
|
|
||||||
disable_data_fsync,
|
|
||||||
meta_device,
|
|
||||||
meta_offset,
|
|
||||||
disable_meta_fsync,
|
|
||||||
disable_device_lock,
|
|
||||||
// blockstore - configurable offline
|
|
||||||
inmemory_metadata,
|
|
||||||
inmemory_journal,
|
|
||||||
journal_sector_buffer_count,
|
|
||||||
journal_no_same_sector_overwrites,
|
|
||||||
// blockstore - configurable online
|
|
||||||
max_write_iodepth,
|
|
||||||
min_flusher_count: 1,
|
|
||||||
max_flusher_count: 256,
|
|
||||||
throttle_small_writes: false,
|
|
||||||
throttle_target_iops: 100,
|
|
||||||
throttle_target_mbs: 100,
|
|
||||||
throttle_target_parallelism: 1,
|
|
||||||
throttle_threshold_us: 50,
|
|
||||||
}, */
|
|
||||||
global: {},
|
|
||||||
/* node_placement: {
|
|
||||||
host1: { level: 'host', parent: 'rack1' },
|
|
||||||
...
|
|
||||||
}, */
|
|
||||||
node_placement: {},
|
|
||||||
/* pools: {
|
|
||||||
<id>: {
|
|
||||||
name: 'testpool',
|
|
||||||
// 'ec' uses Reed-Solomon-Vandermonde codes, 'jerasure' is an alias for 'ec'
|
|
||||||
scheme: 'replicated' | 'xor' | 'ec' | 'jerasure',
|
|
||||||
pg_size: 3,
|
|
||||||
pg_minsize: 2,
|
|
||||||
// number of parity chunks, required for EC
|
|
||||||
parity_chunks?: 1,
|
|
||||||
pg_count: 100,
|
|
||||||
// default is failure_domain=host
|
|
||||||
failure_domain?: 'host',
|
|
||||||
// additional failure domain rules; failure_domain=x is equivalent to x=123..N
|
|
||||||
level_placement?: 'dc=112233 host=123456',
|
|
||||||
raw_placement?: 'any, dc=1 host!=1, dc=1 host!=(1,2)',
|
|
||||||
old_combinator: false,
|
|
||||||
max_osd_combinations: 10000,
|
|
||||||
// block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
|
|
||||||
block_size: 131072,
|
|
||||||
bitmap_granularity: 4096,
|
|
||||||
// 'all'/'small'/'none', same as in OSD options
|
|
||||||
immediate_commit: 'all',
|
|
||||||
pg_stripe_size: 0,
|
|
||||||
root_node?: 'rack1',
|
|
||||||
// restrict pool to OSDs having all of these tags
|
|
||||||
osd_tags?: 'nvme' | [ 'nvme', ... ],
|
|
||||||
// prefer to put primary on OSD with these tags
|
|
||||||
primary_affinity_tags?: 'nvme' | [ 'nvme', ... ],
|
|
||||||
// scrub interval
|
|
||||||
scrub_interval?: '30d',
|
|
||||||
},
|
|
||||||
...
|
|
||||||
}, */
|
|
||||||
pools: {},
|
|
||||||
osd: {
|
|
||||||
/* <id>: { reweight?: 1, tags?: [ 'nvme', ... ], noout?: true }, ... */
|
|
||||||
},
|
|
||||||
/* inode: {
|
|
||||||
<pool_id>: {
|
|
||||||
<inode_t>: {
|
|
||||||
name: string,
|
|
||||||
size?: uint64_t, // bytes
|
|
||||||
parent_pool?: <pool_id>,
|
|
||||||
parent_id?: <inode_t>,
|
|
||||||
readonly?: boolean,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, */
|
|
||||||
inode: {},
|
|
||||||
},
|
|
||||||
osd: {
|
|
||||||
state: {
|
|
||||||
/* <osd_num_t>: {
|
|
||||||
state: "up",
|
|
||||||
addresses: string[],
|
|
||||||
host: string,
|
|
||||||
port: uint16_t,
|
|
||||||
primary_enabled: boolean,
|
|
||||||
blockstore_enabled: boolean,
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
stats: {
|
|
||||||
/* <osd_num_t>: {
|
|
||||||
time: number, // unix time
|
|
||||||
data_block_size: uint64_t, // bytes
|
|
||||||
bitmap_granularity: uint64_t, // bytes
|
|
||||||
immediate_commit: "all"|"small"|"none",
|
|
||||||
blockstore_ready: boolean,
|
|
||||||
size: uint64_t, // bytes
|
|
||||||
free: uint64_t, // bytes
|
|
||||||
host: string,
|
|
||||||
op_stats: {
|
|
||||||
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
||||||
},
|
|
||||||
subop_stats: {
|
|
||||||
<string>: { count: uint64_t, usec: uint64_t },
|
|
||||||
},
|
|
||||||
recovery_stats: {
|
|
||||||
degraded: { count: uint64_t, bytes: uint64_t },
|
|
||||||
misplaced: { count: uint64_t, bytes: uint64_t },
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
inodestats: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
<inode_t>: {
|
|
||||||
read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
||||||
write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
||||||
delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
space: {
|
|
||||||
/* <osd_num_t>: {
|
|
||||||
<pool_id>: {
|
|
||||||
<inode_t>: uint64_t, // bytes
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
mon: {
|
|
||||||
master: {
|
|
||||||
/* ip: [ string ], id: uint64_t */
|
|
||||||
},
|
|
||||||
member: {
|
|
||||||
/* <uint64_t>: { ip: [ string ] }, */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
pg: {
|
|
||||||
/* config: {
|
|
||||||
hash: string,
|
|
||||||
items: {
|
|
||||||
<pool_id>: {
|
|
||||||
<pg_id>: {
|
|
||||||
osd_set: [ 1, 2, 3 ],
|
|
||||||
primary: 1,
|
|
||||||
pause: false,
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}, */
|
|
||||||
config: {},
|
|
||||||
state: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
<pg_id>: {
|
|
||||||
primary: osd_num_t,
|
|
||||||
state: ("starting"|"peering"|"incomplete"|"active"|"repeering"|"stopping"|"offline"|
|
|
||||||
"degraded"|"has_incomplete"|"has_degraded"|"has_misplaced"|"has_unclean"|
|
|
||||||
"has_invalid"|"has_inconsistent"|"has_corrupted"|"left_on_dead"|"scrubbing")[],
|
|
||||||
}
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
history: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
<pg_id>: {
|
|
||||||
osd_sets: osd_num_t[][],
|
|
||||||
all_peers: osd_num_t[],
|
|
||||||
epoch: uint64_t,
|
|
||||||
next_scrub: uint64_t,
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
pgstats: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
<pg_id>: {
|
|
||||||
object_count: uint64_t,
|
|
||||||
clean_count: uint64_t,
|
|
||||||
misplaced_count: uint64_t,
|
|
||||||
degraded_count: uint64_t,
|
|
||||||
incomplete_count: uint64_t,
|
|
||||||
write_osd_set: osd_num_t[],
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
inode: {
|
|
||||||
stats: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
<inode_t>: {
|
|
||||||
raw_used: uint64_t, // raw used bytes on OSDs
|
|
||||||
read: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
||||||
write: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
||||||
delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
||||||
},
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
pool: {
|
|
||||||
stats: {
|
|
||||||
/* <pool_id>: {
|
|
||||||
used_raw_tb: float, // used raw space in the pool
|
|
||||||
total_raw_tb: float, // maximum amount of space in the pool
|
|
||||||
raw_to_usable: float, // raw to usable ratio
|
|
||||||
space_efficiency: float, // 0..1
|
|
||||||
} */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
stats: {
|
|
||||||
/* op_stats: {
|
|
||||||
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
||||||
},
|
|
||||||
subop_stats: {
|
|
||||||
<string>: { count: uint64_t, usec: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
||||||
},
|
|
||||||
recovery_stats: {
|
|
||||||
degraded: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
|
|
||||||
misplaced: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
|
|
||||||
},
|
|
||||||
object_counts: {
|
|
||||||
object: uint64_t,
|
|
||||||
clean: uint64_t,
|
|
||||||
misplaced: uint64_t,
|
|
||||||
degraded: uint64_t,
|
|
||||||
incomplete: uint64_t,
|
|
||||||
},
|
|
||||||
object_bytes: {
|
|
||||||
total: uint64_t,
|
|
||||||
clean: uint64_t,
|
|
||||||
misplaced: uint64_t,
|
|
||||||
degraded: uint64_t,
|
|
||||||
incomplete: uint64_t,
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
history: {
|
|
||||||
last_clean_pgs: {},
|
|
||||||
},
|
|
||||||
index: {
|
|
||||||
image: {
|
|
||||||
/* <name>: {
|
|
||||||
id: uint64_t,
|
|
||||||
pool_id: uint64_t,
|
|
||||||
}, */
|
|
||||||
},
|
|
||||||
maxid: {
|
|
||||||
/* <pool_id>: uint64_t, */
|
|
||||||
},
|
|
||||||
},
|
|
||||||
};
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
etcd_nonempty_keys,
|
|
||||||
etcd_allow,
|
|
||||||
etcd_tree,
|
|
||||||
};
|
|
|
@ -1,50 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const fsp = require('fs').promises;
|
|
||||||
const http = require('http');
|
|
||||||
const https = require('https');
|
|
||||||
|
|
||||||
async function create_http_server(cfg, handler)
|
|
||||||
{
|
|
||||||
let server;
|
|
||||||
if (cfg.mon_https_cert)
|
|
||||||
{
|
|
||||||
const tls = {
|
|
||||||
key: await fsp.readFile(cfg.mon_https_key),
|
|
||||||
cert: await fsp.readFile(cfg.mon_https_cert),
|
|
||||||
};
|
|
||||||
if (cfg.mon_https_ca)
|
|
||||||
{
|
|
||||||
tls.mon_https_ca = await fsp.readFile(cfg.mon_https_ca);
|
|
||||||
}
|
|
||||||
if (cfg.mon_https_client_auth)
|
|
||||||
{
|
|
||||||
tls.requestCert = true;
|
|
||||||
}
|
|
||||||
server = https.createServer(tls, handler);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
server = http.createServer(handler);
|
|
||||||
}
|
|
||||||
try
|
|
||||||
{
|
|
||||||
let err;
|
|
||||||
server.once('error', e => err = e);
|
|
||||||
server.listen(cfg.mon_http_port || 8060, cfg.mon_http_ip || undefined);
|
|
||||||
if (err)
|
|
||||||
throw err;
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
console.error(
|
|
||||||
'HTTP server disabled because listen at address: '+
|
|
||||||
(cfg.mon_http_ip || '')+':'+(cfg.mon_http_port || 9090)+' failed with error: '+e
|
|
||||||
);
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
return server;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = { create_http_server };
|
|
|
@ -23,4 +23,4 @@ for (let i = 2; i < process.argv.length; i++)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
Mon.run_forever(options).catch(console.error);
|
new Mon(options).start().catch(e => { console.error(e); process.exit(1); });
|
||||||
|
|
2080
mon/mon.js
2080
mon/mon.js
File diff suppressed because it is too large
Load Diff
215
mon/osd_tree.js
215
mon/osd_tree.js
|
@ -1,215 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
function get_osd_tree(global_config, state)
|
|
||||||
{
|
|
||||||
const levels = global_config.placement_levels||{};
|
|
||||||
levels.host = levels.host || 100;
|
|
||||||
levels.osd = levels.osd || 101;
|
|
||||||
const tree = {};
|
|
||||||
let up_osds = {};
|
|
||||||
// This requires monitor system time to be in sync with OSD system times (at least to some extent)
|
|
||||||
const down_time = Date.now()/1000 - global_config.osd_out_time;
|
|
||||||
for (const osd_num of Object.keys(state.osd.stats).sort((a, b) => a - b))
|
|
||||||
{
|
|
||||||
const stat = state.osd.stats[osd_num];
|
|
||||||
const osd_cfg = state.config.osd[osd_num];
|
|
||||||
let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
|
|
||||||
if (reweight < 0 || isNaN(reweight))
|
|
||||||
reweight = 1;
|
|
||||||
if (stat && stat.size && reweight && (state.osd.state[osd_num] || Number(stat.time) >= down_time ||
|
|
||||||
osd_cfg && osd_cfg.noout))
|
|
||||||
{
|
|
||||||
// Numeric IDs are reserved for OSDs
|
|
||||||
if (state.osd.state[osd_num] && reweight > 0)
|
|
||||||
{
|
|
||||||
// React to down OSDs immediately
|
|
||||||
up_osds[osd_num] = true;
|
|
||||||
}
|
|
||||||
tree[osd_num] = tree[osd_num] || {};
|
|
||||||
tree[osd_num].id = osd_num;
|
|
||||||
tree[osd_num].parent = tree[osd_num].parent || stat.host;
|
|
||||||
tree[osd_num].level = 'osd';
|
|
||||||
tree[osd_num].size = reweight * stat.size / 1024 / 1024 / 1024 / 1024; // terabytes
|
|
||||||
if (osd_cfg && osd_cfg.tags)
|
|
||||||
{
|
|
||||||
tree[osd_num].tags = (osd_cfg.tags instanceof Array ? [ ...osd_cfg.tags ] : [ osd_cfg.tags ])
|
|
||||||
.reduce((a, c) => { a[c] = true; return a; }, {});
|
|
||||||
}
|
|
||||||
delete tree[osd_num].children;
|
|
||||||
if (!tree[stat.host])
|
|
||||||
{
|
|
||||||
tree[stat.host] = {
|
|
||||||
id: stat.host,
|
|
||||||
level: 'host',
|
|
||||||
parent: null,
|
|
||||||
children: [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const node_id in state.config.node_placement||{})
|
|
||||||
{
|
|
||||||
const node_cfg = state.config.node_placement[node_id];
|
|
||||||
if (/^\d+$/.exec(node_id))
|
|
||||||
{
|
|
||||||
node_cfg.level = 'osd';
|
|
||||||
}
|
|
||||||
if (!node_id || !node_cfg.level || !levels[node_cfg.level] ||
|
|
||||||
node_cfg.level === 'osd' && !tree[node_id])
|
|
||||||
{
|
|
||||||
// All nodes must have non-empty IDs and valid levels
|
|
||||||
// OSDs have to actually exist
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
tree[node_id] = tree[node_id] || {};
|
|
||||||
tree[node_id].id = node_id;
|
|
||||||
tree[node_id].level = node_cfg.level;
|
|
||||||
tree[node_id].parent = node_cfg.parent;
|
|
||||||
if (node_cfg.level !== 'osd')
|
|
||||||
{
|
|
||||||
tree[node_id].children = [];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return { up_osds, levels, osd_tree: tree };
|
|
||||||
}
|
|
||||||
|
|
||||||
function make_hier_tree(global_config, tree)
|
|
||||||
{
|
|
||||||
const levels = global_config.placement_levels||{};
|
|
||||||
levels.host = levels.host || 100;
|
|
||||||
levels.osd = levels.osd || 101;
|
|
||||||
tree = { ...tree };
|
|
||||||
for (const node_id in tree)
|
|
||||||
{
|
|
||||||
tree[node_id] = { ...tree[node_id], children: [] };
|
|
||||||
}
|
|
||||||
tree[''] = { children: [] };
|
|
||||||
for (const node_id in tree)
|
|
||||||
{
|
|
||||||
if (node_id === '' || tree[node_id].level === 'osd' && (!tree[node_id].size || tree[node_id].size <= 0))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const node_cfg = tree[node_id];
|
|
||||||
const node_level = levels[node_cfg.level] || node_cfg.level;
|
|
||||||
let parent_level = node_cfg.parent && tree[node_cfg.parent] && tree[node_cfg.parent].children
|
|
||||||
&& tree[node_cfg.parent].level;
|
|
||||||
parent_level = parent_level ? (levels[parent_level] || parent_level) : null;
|
|
||||||
// Parent's level must be less than child's; OSDs must be leaves
|
|
||||||
const parent = parent_level && parent_level < node_level ? node_cfg.parent : '';
|
|
||||||
tree[parent].children.push(tree[node_id]);
|
|
||||||
}
|
|
||||||
// Delete empty nodes
|
|
||||||
let deleted = 0;
|
|
||||||
do
|
|
||||||
{
|
|
||||||
deleted = 0;
|
|
||||||
for (const node_id in tree)
|
|
||||||
{
|
|
||||||
if (tree[node_id].level !== 'osd' && (!tree[node_id].children || !tree[node_id].children.length))
|
|
||||||
{
|
|
||||||
const parent = tree[node_id].parent;
|
|
||||||
if (parent)
|
|
||||||
{
|
|
||||||
tree[parent].children = tree[parent].children.filter(c => c != tree[node_id]);
|
|
||||||
}
|
|
||||||
deleted++;
|
|
||||||
delete tree[node_id];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} while (deleted > 0);
|
|
||||||
return tree;
|
|
||||||
}
|
|
||||||
|
|
||||||
function filter_osds_by_root_node(global_config, pool_tree, root_node)
|
|
||||||
{
|
|
||||||
if (!root_node)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let hier_tree = make_hier_tree(global_config, pool_tree);
|
|
||||||
let included = [ ...(hier_tree[root_node] || {}).children||[] ];
|
|
||||||
for (let i = 0; i < included.length; i++)
|
|
||||||
{
|
|
||||||
if (included[i].children)
|
|
||||||
{
|
|
||||||
included.splice(i+1, 0, ...included[i].children);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let cur = pool_tree[root_node] || {};
|
|
||||||
while (cur && cur.id)
|
|
||||||
{
|
|
||||||
included.unshift(cur);
|
|
||||||
cur = pool_tree[cur.parent||''];
|
|
||||||
}
|
|
||||||
included = included.reduce((a, c) => { a[c.id||''] = true; return a; }, {});
|
|
||||||
for (const item in pool_tree)
|
|
||||||
{
|
|
||||||
if (!included[item])
|
|
||||||
{
|
|
||||||
delete pool_tree[item];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function filter_osds_by_tags(orig_tree, tags)
|
|
||||||
{
|
|
||||||
if (!tags)
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
for (const tag of (tags instanceof Array ? tags : [ tags ]))
|
|
||||||
{
|
|
||||||
for (const osd in orig_tree)
|
|
||||||
{
|
|
||||||
if (orig_tree[osd].level === 'osd' &&
|
|
||||||
(!orig_tree[osd].tags || !orig_tree[osd].tags[tag]))
|
|
||||||
{
|
|
||||||
delete orig_tree[osd];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function filter_osds_by_block_layout(orig_tree, osd_stats, block_size, bitmap_granularity, immediate_commit)
|
|
||||||
{
|
|
||||||
for (const osd in orig_tree)
|
|
||||||
{
|
|
||||||
if (orig_tree[osd].level === 'osd')
|
|
||||||
{
|
|
||||||
const osd_stat = osd_stats[osd];
|
|
||||||
if (osd_stat && (osd_stat.bs_block_size && osd_stat.bs_block_size != block_size ||
|
|
||||||
osd_stat.bitmap_granularity && osd_stat.bitmap_granularity != bitmap_granularity ||
|
|
||||||
osd_stat.immediate_commit == 'small' && immediate_commit == 'all' ||
|
|
||||||
osd_stat.immediate_commit == 'none' && immediate_commit != 'none'))
|
|
||||||
{
|
|
||||||
delete orig_tree[osd];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
function get_affinity_osds(pool_cfg, up_osds, osd_tree)
|
|
||||||
{
|
|
||||||
let aff_osds = up_osds;
|
|
||||||
if (pool_cfg.primary_affinity_tags)
|
|
||||||
{
|
|
||||||
aff_osds = Object.keys(up_osds).reduce((a, c) => { a[c] = osd_tree[c]; return a; }, {});
|
|
||||||
filter_osds_by_tags(aff_osds, pool_cfg.primary_affinity_tags);
|
|
||||||
for (const osd in aff_osds)
|
|
||||||
{
|
|
||||||
aff_osds[osd] = true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return aff_osds;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
get_osd_tree,
|
|
||||||
make_hier_tree,
|
|
||||||
filter_osds_by_root_node,
|
|
||||||
filter_osds_by_tags,
|
|
||||||
filter_osds_by_block_layout,
|
|
||||||
get_affinity_osds,
|
|
||||||
};
|
|
|
@ -1,24 +1,25 @@
|
||||||
{
|
{
|
||||||
"name": "vitastor-mon",
|
"name": "vitastor-mon",
|
||||||
"version": "1.9.1",
|
"version": "1.6.1",
|
||||||
"description": "Vitastor SDS monitor service",
|
"description": "Vitastor SDS monitor service",
|
||||||
"main": "mon-main.js",
|
"main": "mon-main.js",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"lint": "eslint *.js lp_optimizer/*.js scripts/*.js"
|
"test": "echo \"Error: no test specified\" && exit 1"
|
||||||
},
|
},
|
||||||
"author": "Vitaliy Filippov",
|
"author": "Vitaliy Filippov",
|
||||||
"license": "UNLICENSED",
|
"license": "UNLICENSED",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"antietcd": "^1.1.0",
|
|
||||||
"sprintf-js": "^1.1.2",
|
"sprintf-js": "^1.1.2",
|
||||||
"ws": "^7.2.5"
|
"ws": "^7.2.5"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"eslint": "^8.0.0",
|
"eslint": "^8.0.0",
|
||||||
"eslint-plugin-import": "^2.29.1",
|
|
||||||
"eslint-plugin-node": "^11.1.0"
|
"eslint-plugin-node": "^11.1.0"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=12.0.0"
|
"node": ">=12.0.0"
|
||||||
|
},
|
||||||
|
"scripts": {
|
||||||
|
"lint": "eslint *.js"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
267
mon/pg_gen.js
267
mon/pg_gen.js
|
@ -1,267 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const { RuleCombinator } = require('./lp_optimizer/dsl_pgs.js');
|
|
||||||
const { SimpleCombinator, flatten_tree } = require('./lp_optimizer/simple_pgs.js');
|
|
||||||
const { validate_pool_cfg, get_pg_rules } = require('./pool_config.js');
|
|
||||||
const LPOptimizer = require('./lp_optimizer/lp_optimizer.js');
|
|
||||||
const { scale_pg_count } = require('./pg_utils.js');
|
|
||||||
const { make_hier_tree, filter_osds_by_root_node,
|
|
||||||
filter_osds_by_tags, filter_osds_by_block_layout, get_affinity_osds } = require('./osd_tree.js');
|
|
||||||
|
|
||||||
let seed;
|
|
||||||
|
|
||||||
function reset_rng()
|
|
||||||
{
|
|
||||||
seed = 0x5f020e43;
|
|
||||||
}
|
|
||||||
|
|
||||||
function rng()
|
|
||||||
{
|
|
||||||
seed ^= seed << 13;
|
|
||||||
seed ^= seed >> 17;
|
|
||||||
seed ^= seed << 5;
|
|
||||||
return seed + 2147483648;
|
|
||||||
}
|
|
||||||
|
|
||||||
function pick_primary(pool_config, osd_set, up_osds, aff_osds)
|
|
||||||
{
|
|
||||||
let alive_set;
|
|
||||||
if (pool_config.scheme === 'replicated')
|
|
||||||
{
|
|
||||||
// Prefer "affinity" OSDs
|
|
||||||
alive_set = osd_set.filter(osd_num => osd_num && aff_osds[osd_num]);
|
|
||||||
if (!alive_set.length)
|
|
||||||
alive_set = osd_set.filter(osd_num => osd_num && up_osds[osd_num]);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
// Prefer data OSDs for EC because they can actually read something without an additional network hop
|
|
||||||
const pg_data_size = (pool_config.pg_size||0) - (pool_config.parity_chunks||0);
|
|
||||||
alive_set = osd_set.slice(0, pg_data_size).filter(osd_num => osd_num && aff_osds[osd_num]);
|
|
||||||
if (!alive_set.length)
|
|
||||||
alive_set = osd_set.filter(osd_num => osd_num && aff_osds[osd_num]);
|
|
||||||
if (!alive_set.length)
|
|
||||||
{
|
|
||||||
alive_set = osd_set.slice(0, pg_data_size).filter(osd_num => osd_num && up_osds[osd_num]);
|
|
||||||
if (!alive_set.length)
|
|
||||||
alive_set = osd_set.filter(osd_num => osd_num && up_osds[osd_num]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (!alive_set.length)
|
|
||||||
{
|
|
||||||
return 0;
|
|
||||||
}
|
|
||||||
return alive_set[rng() % alive_set.length];
|
|
||||||
}
|
|
||||||
|
|
||||||
function recheck_primary(state, global_config, up_osds, osd_tree)
|
|
||||||
{
|
|
||||||
let new_pg_config;
|
|
||||||
for (const pool_id in state.config.pools)
|
|
||||||
{
|
|
||||||
const pool_cfg = state.config.pools[pool_id];
|
|
||||||
if (!validate_pool_cfg(pool_id, pool_cfg, global_config.placement_levels, false))
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const aff_osds = get_affinity_osds(pool_cfg, up_osds, osd_tree);
|
|
||||||
reset_rng();
|
|
||||||
for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
|
|
||||||
{
|
|
||||||
if (!state.pg.config.items[pool_id])
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
const pg_cfg = state.pg.config.items[pool_id][pg_num];
|
|
||||||
if (pg_cfg)
|
|
||||||
{
|
|
||||||
const new_primary = pick_primary(state.config.pools[pool_id], pg_cfg.osd_set, up_osds, aff_osds);
|
|
||||||
if (pg_cfg.primary != new_primary)
|
|
||||||
{
|
|
||||||
if (!new_pg_config)
|
|
||||||
{
|
|
||||||
new_pg_config = JSON.parse(JSON.stringify(state.pg.config));
|
|
||||||
}
|
|
||||||
console.log(
|
|
||||||
`Moving pool ${pool_id} (${pool_cfg.name || 'unnamed'}) PG ${pg_num}`+
|
|
||||||
` primary OSD from ${pg_cfg.primary} to ${new_primary}`
|
|
||||||
);
|
|
||||||
new_pg_config.items[pool_id][pg_num].primary = new_primary;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return new_pg_config;
|
|
||||||
}
|
|
||||||
|
|
||||||
function save_new_pgs_txn(save_to, request, state, etcd_prefix, etcd_watch_revision, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history)
|
|
||||||
{
|
|
||||||
const aff_osds = get_affinity_osds(state.config.pools[pool_id] || {}, up_osds, osd_tree);
|
|
||||||
const pg_items = {};
|
|
||||||
reset_rng();
|
|
||||||
new_pgs.map((osd_set, i) =>
|
|
||||||
{
|
|
||||||
osd_set = osd_set.map(osd_num => osd_num === LPOptimizer.NO_OSD ? 0 : osd_num);
|
|
||||||
pg_items[i+1] = {
|
|
||||||
osd_set,
|
|
||||||
primary: pick_primary(state.config.pools[pool_id], osd_set, up_osds, aff_osds),
|
|
||||||
};
|
|
||||||
if (prev_pgs[i] && prev_pgs[i].join(' ') != osd_set.join(' ') &&
|
|
||||||
prev_pgs[i].filter(osd_num => osd_num).length > 0)
|
|
||||||
{
|
|
||||||
pg_history[i] = pg_history[i] || {};
|
|
||||||
pg_history[i].osd_sets = pg_history[i].osd_sets || [];
|
|
||||||
pg_history[i].osd_sets.push(prev_pgs[i]);
|
|
||||||
}
|
|
||||||
if (pg_history[i] && pg_history[i].osd_sets)
|
|
||||||
{
|
|
||||||
pg_history[i].osd_sets = Object.values(pg_history[i].osd_sets
|
|
||||||
.reduce((a, c) => { a[c.join(' ')] = c; return a; }, {}));
|
|
||||||
}
|
|
||||||
});
|
|
||||||
for (let i = 0; i < new_pgs.length || i < prev_pgs.length; i++)
|
|
||||||
{
|
|
||||||
// FIXME: etcd has max_txn_ops limit, and it's 128 by default
|
|
||||||
// Sooo we probably want to change our storage scheme for PG histories...
|
|
||||||
request.compare.push({
|
|
||||||
key: b64(etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
|
||||||
target: 'MOD',
|
|
||||||
mod_revision: ''+etcd_watch_revision,
|
|
||||||
result: 'LESS',
|
|
||||||
});
|
|
||||||
if (pg_history[i])
|
|
||||||
{
|
|
||||||
request.success.push({
|
|
||||||
requestPut: {
|
|
||||||
key: b64(etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
|
||||||
value: b64(JSON.stringify(pg_history[i])),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
request.success.push({
|
|
||||||
requestDeleteRange: {
|
|
||||||
key: b64(etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
|
||||||
}
|
|
||||||
save_to.items = save_to.items || {};
|
|
||||||
if (!new_pgs.length)
|
|
||||||
{
|
|
||||||
delete save_to.items[pool_id];
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
save_to.items[pool_id] = pg_items;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function generate_pool_pgs(state, global_config, pool_id, osd_tree, levels)
|
|
||||||
{
|
|
||||||
const pool_cfg = state.config.pools[pool_id];
|
|
||||||
if (!validate_pool_cfg(pool_id, pool_cfg, global_config.placement_levels, false))
|
|
||||||
{
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
let pool_tree = { ...osd_tree };
|
|
||||||
filter_osds_by_root_node(global_config, pool_tree, pool_cfg.root_node);
|
|
||||||
filter_osds_by_tags(pool_tree, pool_cfg.osd_tags);
|
|
||||||
filter_osds_by_block_layout(
|
|
||||||
pool_tree,
|
|
||||||
state.osd.stats,
|
|
||||||
pool_cfg.block_size || global_config.block_size || 131072,
|
|
||||||
pool_cfg.bitmap_granularity || global_config.bitmap_granularity || 4096,
|
|
||||||
pool_cfg.immediate_commit || global_config.immediate_commit || 'all'
|
|
||||||
);
|
|
||||||
pool_tree = make_hier_tree(global_config, pool_tree);
|
|
||||||
// First try last_clean_pgs to minimize data movement
|
|
||||||
let prev_pgs = [];
|
|
||||||
for (const pg in ((state.history.last_clean_pgs.items||{})[pool_id]||{}))
|
|
||||||
{
|
|
||||||
prev_pgs[pg-1] = [ ...state.history.last_clean_pgs.items[pool_id][pg].osd_set ];
|
|
||||||
}
|
|
||||||
if (!prev_pgs.length)
|
|
||||||
{
|
|
||||||
// Fall back to pg/config if it's empty
|
|
||||||
for (const pg in ((state.pg.config.items||{})[pool_id]||{}))
|
|
||||||
{
|
|
||||||
prev_pgs[pg-1] = [ ...state.pg.config.items[pool_id][pg].osd_set ];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const old_pg_count = prev_pgs.length;
|
|
||||||
const optimize_cfg = {
|
|
||||||
osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}),
|
|
||||||
combinator: !global_config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement
|
|
||||||
// new algorithm:
|
|
||||||
? new RuleCombinator(pool_tree, get_pg_rules(pool_id, pool_cfg, global_config.placement_levels), pool_cfg.max_osd_combinations)
|
|
||||||
// old algorithm:
|
|
||||||
: new SimpleCombinator(flatten_tree(pool_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
|
|
||||||
pg_count: pool_cfg.pg_count,
|
|
||||||
pg_size: pool_cfg.pg_size,
|
|
||||||
pg_minsize: pool_cfg.pg_minsize,
|
|
||||||
ordered: pool_cfg.scheme != 'replicated',
|
|
||||||
};
|
|
||||||
let optimize_result;
|
|
||||||
// Re-shuffle PGs if pg/config.hash is empty
|
|
||||||
if (old_pg_count > 0 && state.pg.config.hash)
|
|
||||||
{
|
|
||||||
if (prev_pgs.length != pool_cfg.pg_count)
|
|
||||||
{
|
|
||||||
// Scale PG count
|
|
||||||
// Do it even if old_pg_count is already equal to pool_cfg.pg_count,
|
|
||||||
// because last_clean_pgs may still contain the old number of PGs
|
|
||||||
scale_pg_count(prev_pgs, pool_cfg.pg_count);
|
|
||||||
}
|
|
||||||
for (const pg of prev_pgs)
|
|
||||||
{
|
|
||||||
while (pg.length < pool_cfg.pg_size)
|
|
||||||
{
|
|
||||||
pg.push(0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
optimize_result = await LPOptimizer.optimize_change({
|
|
||||||
prev_pgs,
|
|
||||||
...optimize_cfg,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
optimize_result = await LPOptimizer.optimize_initial(optimize_cfg);
|
|
||||||
}
|
|
||||||
console.log(`Pool ${pool_id} (${pool_cfg.name || 'unnamed'}):`);
|
|
||||||
LPOptimizer.print_change_stats(optimize_result);
|
|
||||||
let pg_effsize = pool_cfg.pg_size;
|
|
||||||
for (const pg of optimize_result.int_pgs)
|
|
||||||
{
|
|
||||||
const this_pg_size = pg.filter(osd => osd != LPOptimizer.NO_OSD).length;
|
|
||||||
if (this_pg_size && this_pg_size < pg_effsize)
|
|
||||||
{
|
|
||||||
pg_effsize = this_pg_size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return {
|
|
||||||
pool_id,
|
|
||||||
pgs: optimize_result.int_pgs,
|
|
||||||
stats: {
|
|
||||||
total_raw_tb: optimize_result.space,
|
|
||||||
pg_real_size: pg_effsize || pool_cfg.pg_size,
|
|
||||||
raw_to_usable: (pg_effsize || pool_cfg.pg_size) / (pool_cfg.scheme === 'replicated'
|
|
||||||
? 1 : (pool_cfg.pg_size - (pool_cfg.parity_chunks||0))),
|
|
||||||
space_efficiency: optimize_result.space/(optimize_result.total_space||1),
|
|
||||||
},
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
function b64(str)
|
|
||||||
{
|
|
||||||
return Buffer.from(str).toString('base64');
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
recheck_primary,
|
|
||||||
save_new_pgs_txn,
|
|
||||||
generate_pool_pgs,
|
|
||||||
};
|
|
|
@ -1,169 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const { parse_level_indexes, parse_pg_dsl } = require('./lp_optimizer/dsl_pgs.js');
|
|
||||||
|
|
||||||
function validate_pool_cfg(pool_id, pool_cfg, placement_levels, warn)
|
|
||||||
{
|
|
||||||
pool_cfg.pg_size = Math.floor(pool_cfg.pg_size);
|
|
||||||
pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
|
|
||||||
pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
|
|
||||||
pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
|
|
||||||
pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
|
|
||||||
if (!/^[1-9]\d*$/.exec(''+pool_id))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool ID '+pool_id+' is invalid');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.scheme !== 'xor' && pool_cfg.scheme !== 'replicated' &&
|
|
||||||
pool_cfg.scheme !== 'ec' && pool_cfg.scheme !== 'jerasure')
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid coding scheme (one of "xor", "replicated", "ec" and "jerasure" required)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!pool_cfg.pg_size || pool_cfg.pg_size < 1 || pool_cfg.pg_size > 256 ||
|
|
||||||
pool_cfg.scheme !== 'replicated' && pool_cfg.pg_size < 3)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid pg_size');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!pool_cfg.pg_minsize || pool_cfg.pg_minsize < 1 || pool_cfg.pg_minsize > pool_cfg.pg_size ||
|
|
||||||
pool_cfg.scheme === 'xor' && pool_cfg.pg_minsize < (pool_cfg.pg_size - 1))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid pg_minsize');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.scheme === 'xor' && pool_cfg.parity_chunks != 0 && pool_cfg.parity_chunks != 1)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid parity_chunks (must be 1)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if ((pool_cfg.scheme === 'ec' || pool_cfg.scheme === 'jerasure') &&
|
|
||||||
(pool_cfg.parity_chunks < 1 || pool_cfg.parity_chunks > pool_cfg.pg_size-2))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid parity_chunks (must be between 1 and pg_size-2)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!pool_cfg.pg_count || pool_cfg.pg_count < 1)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid pg_count');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!pool_cfg.name)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has empty name');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.max_osd_combinations < 100)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid max_osd_combinations (must be at least 100)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.root_node && typeof(pool_cfg.root_node) != 'string')
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid root_node (must be a string)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.osd_tags && typeof(pool_cfg.osd_tags) != 'string' &&
|
|
||||||
(!(pool_cfg.osd_tags instanceof Array) || pool_cfg.osd_tags.filter(t => typeof t != 'string').length > 0))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid osd_tags (must be a string or array of strings)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (pool_cfg.primary_affinity_tags && typeof(pool_cfg.primary_affinity_tags) != 'string' &&
|
|
||||||
(!(pool_cfg.primary_affinity_tags instanceof Array) || pool_cfg.primary_affinity_tags.filter(t => typeof t != 'string').length > 0))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' has invalid primary_affinity_tags (must be a string or array of strings)');
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
if (!get_pg_rules(pool_id, pool_cfg, placement_levels, true))
|
|
||||||
{
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
function get_pg_rules(pool_id, pool_cfg, placement_levels, warn)
|
|
||||||
{
|
|
||||||
if (pool_cfg.level_placement)
|
|
||||||
{
|
|
||||||
const pg_size = (0|pool_cfg.pg_size);
|
|
||||||
let rules = pool_cfg.level_placement;
|
|
||||||
if (typeof rules === 'string')
|
|
||||||
{
|
|
||||||
rules = rules.split(/\s+/).map(s => s.split(/=/, 2)).reduce((a, c) => { a[c[0]] = c[1]; return a; }, {});
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
rules = { ...rules };
|
|
||||||
}
|
|
||||||
// Always add failure_domain to prevent rules from being totally incorrect
|
|
||||||
const all_diff = [];
|
|
||||||
for (let i = 1; i <= pg_size; i++)
|
|
||||||
{
|
|
||||||
all_diff.push(i);
|
|
||||||
}
|
|
||||||
rules[pool_cfg.failure_domain || 'host'] = all_diff;
|
|
||||||
placement_levels = placement_levels||{};
|
|
||||||
placement_levels.host = placement_levels.host || 100;
|
|
||||||
placement_levels.osd = placement_levels.osd || 101;
|
|
||||||
for (const k in rules)
|
|
||||||
{
|
|
||||||
if (!placement_levels[k] || typeof rules[k] !== 'string' &&
|
|
||||||
(!(rules[k] instanceof Array) ||
|
|
||||||
rules[k].filter(s => typeof s !== 'string' && typeof s !== 'number').length > 0))
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' configuration is invalid: level_placement should be { [level]: string | (string|number)[] }');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
else if (rules[k].length != pg_size)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' configuration is invalid: values in level_placement should contain exactly pg_size ('+pg_size+') items');
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return parse_level_indexes(rules);
|
|
||||||
}
|
|
||||||
else if (typeof pool_cfg.raw_placement === 'string')
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
return parse_pg_dsl(pool_cfg.raw_placement);
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
if (warn)
|
|
||||||
console.log('Pool '+pool_id+' configuration is invalid: invalid raw_placement: '+e.message);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
let rules = [ [] ];
|
|
||||||
let prev = [ 1 ];
|
|
||||||
for (let i = 1; i < pool_cfg.pg_size; i++)
|
|
||||||
{
|
|
||||||
rules.push([ [ pool_cfg.failure_domain||'host', '!=', prev ] ]);
|
|
||||||
prev = [ ...prev, i+1 ];
|
|
||||||
}
|
|
||||||
return rules;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
validate_pool_cfg,
|
|
||||||
get_pg_rules,
|
|
||||||
};
|
|
|
@ -1,220 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const metric_help =
|
|
||||||
`# HELP vitastor_object_bytes Total size of objects in cluster in bytes
|
|
||||||
# TYPE vitastor_object_bytes gauge
|
|
||||||
# HELP vitastor_object_count Total number of objects in cluster
|
|
||||||
# TYPE vitastor_object_count gauge
|
|
||||||
# HELP vitastor_stat_count Total operation count
|
|
||||||
# TYPE vitastor_stat_count counter
|
|
||||||
# HELP vitastor_stat_usec Total operation latency in usec
|
|
||||||
# TYPE vitastor_stat_usec counter
|
|
||||||
# HELP vitastor_stat_bytes Total operation size in bytes
|
|
||||||
# HELP vitastor_stat_bytes counter
|
|
||||||
|
|
||||||
# HELP vitastor_image_raw_used Image raw used size in bytes
|
|
||||||
# TYPE vitastor_image_raw_used counter
|
|
||||||
# HELP vitastor_image_stat_count Per-image total operation count
|
|
||||||
# TYPE vitastor_image_stat_count counter
|
|
||||||
# HELP vitastor_image_stat_usec Per-image total operation latency
|
|
||||||
# TYPE vitastor_image_stat_usec counter
|
|
||||||
# HELP vitastor_image_stat_bytes Per-image total operation size in bytes
|
|
||||||
# TYPE vitastor_image_stat_bytes counter
|
|
||||||
|
|
||||||
# HELP vitastor_osd_status OSD up/down status
|
|
||||||
# TYPE vitastor_osd_status gauge
|
|
||||||
# HELP vitastor_osd_size_bytes OSD total space in bytes
|
|
||||||
# TYPE vitastor_osd_size_bytes gauge
|
|
||||||
# HELP vitastor_osd_free_bytes OSD free space in bytes
|
|
||||||
# TYPE vitastor_osd_free_bytes gauge
|
|
||||||
# HELP vitastor_osd_stat_count Per-image total operation count
|
|
||||||
# TYPE vitastor_osd_stat_count counter
|
|
||||||
# HELP vitastor_osd_stat_usec Per-image total operation latency
|
|
||||||
# TYPE vitastor_osd_stat_usec counter
|
|
||||||
# HELP vitastor_osd_stat_bytes Per-image total operation size in bytes
|
|
||||||
# TYPE vitastor_osd_stat_bytes counter
|
|
||||||
|
|
||||||
# HELP vitastor_monitor_info Monitor info, 1 is master, 0 is standby
|
|
||||||
# TYPE vitastor_monitor_info gauge
|
|
||||||
|
|
||||||
# HELP vitastor_pool_info Pool configuration (in labels)
|
|
||||||
# TYPE vitastor_pool_info gauge
|
|
||||||
# HELP vitastor_pool_status Pool up/down status
|
|
||||||
# TYPE vitastor_pool_status gauge
|
|
||||||
# HELP vitastor_pool_raw_to_usable Raw to usable space ratio
|
|
||||||
# TYPE vitastor_pool_raw_to_usable gauge
|
|
||||||
# HELP vitastor_pool_space_efficiency Pool space usage efficiency
|
|
||||||
# TYPE vitastor_pool_space_efficiency gauge
|
|
||||||
# HELP vitastor_pool_total_raw_tb Total raw space in pool in TB
|
|
||||||
# TYPE vitastor_pool_total_raw_tb gauge
|
|
||||||
# HELP vitastor_pool_used_raw_tb Used raw space in pool in TB
|
|
||||||
# TYPE vitastor_pool_used_raw_tb gauge
|
|
||||||
# HELP vitastor_pg_count PG counts by state
|
|
||||||
# HELP vitastor_pg_count gauge
|
|
||||||
|
|
||||||
`;
|
|
||||||
|
|
||||||
function export_prometheus_metrics(st)
|
|
||||||
{
|
|
||||||
let res = metric_help;
|
|
||||||
|
|
||||||
// Global statistics
|
|
||||||
|
|
||||||
for (const k in st.stats.object_bytes)
|
|
||||||
{
|
|
||||||
res += `vitastor_object_bytes{object_type="${k}"} ${st.stats.object_bytes[k]}\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const k in st.stats.object_counts)
|
|
||||||
{
|
|
||||||
res += `vitastor_object_count{object_type="${k}"} ${st.stats.object_counts[k]}\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const typ of [ 'op', 'subop', 'recovery' ])
|
|
||||||
{
|
|
||||||
for (const op in st.stats[typ+"_stats"]||{})
|
|
||||||
{
|
|
||||||
const op_stat = st.stats[typ+"_stats"][op];
|
|
||||||
for (const key of [ 'count', 'usec', 'bytes' ])
|
|
||||||
{
|
|
||||||
res += `vitastor_stat_${key}{op="${op}",op_type="${typ}"} ${op_stat[key]||0}\n`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per-image statistics
|
|
||||||
|
|
||||||
for (const pool in st.inode.stats)
|
|
||||||
{
|
|
||||||
for (const inode in st.inode.stats[pool])
|
|
||||||
{
|
|
||||||
const ist = st.inode.stats[pool][inode];
|
|
||||||
const inode_name = ((st.config.inode[pool]||{})[inode]||{}).name||'';
|
|
||||||
const inode_label = `image_name="${addslashes(inode_name)}",inode_num="${inode}",pool_id="${pool}"`;
|
|
||||||
res += `vitastor_image_raw_used{${inode_label}} ${ist.raw_used||0}\n`;
|
|
||||||
for (const op of [ 'read', 'write', 'delete' ])
|
|
||||||
{
|
|
||||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
|
||||||
{
|
|
||||||
if (ist[op])
|
|
||||||
{
|
|
||||||
res += `vitastor_image_stat_${k}{${inode_label},op="${op}"} ${ist[op][k]||0}\n`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per-OSD statistics
|
|
||||||
|
|
||||||
for (const osd in st.osd.stats)
|
|
||||||
{
|
|
||||||
const osd_stat = st.osd.stats[osd];
|
|
||||||
const up = st.osd.state[osd] && st.osd.state[osd].state == 'up' ? 1 : 0;
|
|
||||||
res += `vitastor_osd_status{host="${addslashes(osd_stat.host)}",osd_num="${osd}"} ${up}\n`;
|
|
||||||
res += `vitastor_osd_size_bytes{osd_num="${osd}"} ${osd_stat.size||0}\n`;
|
|
||||||
res += `vitastor_osd_free_bytes{osd_num="${osd}"} ${osd_stat.free||0}\n`;
|
|
||||||
for (const op in osd_stat.op_stats)
|
|
||||||
{
|
|
||||||
const ist = osd_stat.op_stats[op];
|
|
||||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
|
||||||
{
|
|
||||||
res += `vitastor_osd_stat_${k}{osd_num="${osd}",op="${op}",op_type="op"} ${ist[k]||0}\n`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const op in osd_stat.subop_stats)
|
|
||||||
{
|
|
||||||
const ist = osd_stat.subop_stats[op];
|
|
||||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
|
||||||
{
|
|
||||||
res += `vitastor_osd_stat_${k}{osd_num="${osd}",op="${op}",op_type="subop"} ${ist[k]||0}\n`;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Monitor statistics
|
|
||||||
|
|
||||||
for (const mon_id in st.mon.member)
|
|
||||||
{
|
|
||||||
const mon = st.mon.member[mon_id];
|
|
||||||
const master = st.mon.master && st.mon.master.id == mon_id ? 1 : 0;
|
|
||||||
const ip = (mon.ip instanceof Array ? mon.ip[0] : mon.ip) || '';
|
|
||||||
res += `vitastor_monitor_info{monitor_hostname="${addslashes(mon.hostname)}",monitor_id="${mon_id}",monitor_ip="${addslashes(ip)}"} ${master}\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Per-pool statistics
|
|
||||||
|
|
||||||
for (const pool_id in st.config.pools)
|
|
||||||
{
|
|
||||||
const pool_cfg = st.config.pools[pool_id];
|
|
||||||
const pool_label = `pool_id="${pool_id}",pool_name="${addslashes(pool_cfg.name)}"`;
|
|
||||||
const pool_stat = st.pool.stats[pool_id];
|
|
||||||
res += `vitastor_pool_info{${pool_label}`+
|
|
||||||
`,pool_scheme="${addslashes(pool_cfg.scheme)}"`+
|
|
||||||
`,pg_size="${pool_cfg.pg_size||0}",pg_minsize="${pool_cfg.pg_minsize||0}"`+
|
|
||||||
`,parity_chunks="${pool_cfg.parity_chunks||0}",pg_count="${pool_cfg.pg_count||0}"`+
|
|
||||||
`,failure_domain="${addslashes(pool_cfg.failure_domain)}"`+
|
|
||||||
`} 1\n`;
|
|
||||||
if (!pool_stat)
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
res += `vitastor_pool_raw_to_usable{${pool_label}} ${pool_stat.raw_to_usable||0}\n`;
|
|
||||||
res += `vitastor_pool_space_efficiency{${pool_label}} ${pool_stat.space_efficiency||0}\n`;
|
|
||||||
res += `vitastor_pool_total_raw_tb{${pool_label}} ${pool_stat.total_raw_tb||0}\n`;
|
|
||||||
res += `vitastor_pool_used_raw_tb{${pool_label}} ${pool_stat.used_raw_tb||0}\n`;
|
|
||||||
|
|
||||||
// PG states and pool up/down status
|
|
||||||
const real_pg_count = (Object.keys(((st.pg.config||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count);
|
|
||||||
const per_state = {
|
|
||||||
active: 0,
|
|
||||||
starting: 0,
|
|
||||||
peering: 0,
|
|
||||||
incomplete: 0,
|
|
||||||
repeering: 0,
|
|
||||||
stopping: 0,
|
|
||||||
offline: 0,
|
|
||||||
degraded: 0,
|
|
||||||
has_inconsistent: 0,
|
|
||||||
has_corrupted: 0,
|
|
||||||
has_incomplete: 0,
|
|
||||||
has_degraded: 0,
|
|
||||||
has_misplaced: 0,
|
|
||||||
has_unclean: 0,
|
|
||||||
has_invalid: 0,
|
|
||||||
left_on_dead: 0,
|
|
||||||
scrubbing: 0,
|
|
||||||
};
|
|
||||||
const pool_pg_states = st.pg.state[pool_id] || {};
|
|
||||||
for (let i = 1; i <= real_pg_count; i++)
|
|
||||||
{
|
|
||||||
if (!pool_pg_states[i])
|
|
||||||
{
|
|
||||||
per_state['offline'] = 1 + (per_state['offline']|0);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
for (const st_name of pool_pg_states[i].state)
|
|
||||||
{
|
|
||||||
per_state[st_name] = 1 + (per_state[st_name]|0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const st_name in per_state)
|
|
||||||
{
|
|
||||||
res += `vitastor_pg_count{pg_state="${st_name}",${pool_label}} ${per_state[st_name]}\n`;
|
|
||||||
}
|
|
||||||
const pool_active = per_state['active'] >= real_pg_count ? 1 : 0;
|
|
||||||
res += `vitastor_pool_status{${pool_label}} ${pool_active}\n`;
|
|
||||||
}
|
|
||||||
|
|
||||||
return res;
|
|
||||||
}
|
|
||||||
|
|
||||||
function addslashes(str)
|
|
||||||
{
|
|
||||||
return ((str||'')+'').replace(/(["\n\\])/g, "\\$1"); // escape " \n \
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = { export_prometheus_metrics };
|
|
File diff suppressed because it is too large
Load Diff
292
mon/stats.js
292
mon/stats.js
|
@ -1,292 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
function derive_osd_stats(st, prev, prev_diff)
|
|
||||||
{
|
|
||||||
const diff = prev_diff || { op_stats: {}, subop_stats: {}, recovery_stats: {}, inode_stats: {} };
|
|
||||||
if (!st || !st.time || !prev || !prev.time || prev.time >= st.time)
|
|
||||||
{
|
|
||||||
return diff;
|
|
||||||
}
|
|
||||||
const timediff = BigInt(st.time*1000 - prev.time*1000);
|
|
||||||
for (const op in st.op_stats||{})
|
|
||||||
{
|
|
||||||
const pr = prev && prev.op_stats && prev.op_stats[op];
|
|
||||||
let c = st.op_stats[op];
|
|
||||||
c = { bytes: BigInt(c.bytes||0), usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
|
|
||||||
const b = c.bytes - BigInt(pr && pr.bytes||0);
|
|
||||||
const us = c.usec - BigInt(pr && pr.usec||0);
|
|
||||||
const n = c.count - BigInt(pr && pr.count||0);
|
|
||||||
diff.op_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n, lat: n > 0 ? us/n : 0n };
|
|
||||||
}
|
|
||||||
for (const op in st.subop_stats||{})
|
|
||||||
{
|
|
||||||
const pr = prev && prev.subop_stats && prev.subop_stats[op];
|
|
||||||
let c = st.subop_stats[op];
|
|
||||||
c = { usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
|
|
||||||
const us = c.usec - BigInt(pr && pr.usec||0);
|
|
||||||
const n = c.count - BigInt(pr && pr.count||0);
|
|
||||||
diff.subop_stats[op] = { ...c, iops: n > 0 ? n*1000n/timediff : 0n, lat: n > 0 ? us/n : 0n };
|
|
||||||
}
|
|
||||||
for (const op in st.recovery_stats||{})
|
|
||||||
{
|
|
||||||
const pr = prev && prev.recovery_stats && prev.recovery_stats[op];
|
|
||||||
let c = st.recovery_stats[op];
|
|
||||||
c = { bytes: BigInt(c.bytes||0), count: BigInt(c.count||0) };
|
|
||||||
const b = c.bytes - BigInt(pr && pr.bytes||0);
|
|
||||||
const n = c.count - BigInt(pr && pr.count||0);
|
|
||||||
diff.recovery_stats[op] = { ...c, bps: n > 0 ? b*1000n/timediff : 0n, iops: n > 0 ? n*1000n/timediff : 0n };
|
|
||||||
}
|
|
||||||
for (const pool_id in st.inode_stats||{})
|
|
||||||
{
|
|
||||||
diff.inode_stats[pool_id] = {};
|
|
||||||
for (const inode_num in st.inode_stats[pool_id])
|
|
||||||
{
|
|
||||||
const inode_diff = diff.inode_stats[pool_id][inode_num] = {};
|
|
||||||
for (const op of [ 'read', 'write', 'delete' ])
|
|
||||||
{
|
|
||||||
const c = st.inode_stats[pool_id][inode_num][op];
|
|
||||||
const pr = prev && prev.inode_stats && prev.inode_stats[pool_id] &&
|
|
||||||
prev.inode_stats[pool_id][inode_num] && prev.inode_stats[pool_id][inode_num][op];
|
|
||||||
const n = BigInt(c.count||0) - BigInt(pr && pr.count||0);
|
|
||||||
inode_diff[op] = {
|
|
||||||
bps: n > 0 ? (BigInt(c.bytes||0) - BigInt(pr && pr.bytes||0))*1000n/timediff : 0n,
|
|
||||||
iops: n > 0 ? n*1000n/timediff : 0n,
|
|
||||||
lat: n > 0 ? (BigInt(c.usec||0) - BigInt(pr && pr.usec||0))/n : 0n,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return diff;
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum_op_stats(this.state.osd, this.prev_stats)
|
|
||||||
function sum_op_stats(all_osd, prev_stats)
|
|
||||||
{
|
|
||||||
for (const osd in all_osd.stats)
|
|
||||||
{
|
|
||||||
const cur = { ...all_osd.stats[osd], inode_stats: all_osd.inodestats[osd]||{} };
|
|
||||||
prev_stats.osd_diff[osd] = derive_osd_stats(
|
|
||||||
cur, prev_stats.osd_stats[osd], prev_stats.osd_diff[osd]
|
|
||||||
);
|
|
||||||
prev_stats.osd_stats[osd] = cur;
|
|
||||||
}
|
|
||||||
const sum_diff = { op_stats: {}, subop_stats: {}, recovery_stats: { degraded: {}, misplaced: {} } };
|
|
||||||
// Sum derived values instead of deriving summed
|
|
||||||
for (const osd in all_osd.state)
|
|
||||||
{
|
|
||||||
const derived = prev_stats.osd_diff[osd];
|
|
||||||
if (!all_osd.state[osd] || !derived)
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for (const type in sum_diff)
|
|
||||||
{
|
|
||||||
for (const op in derived[type]||{})
|
|
||||||
{
|
|
||||||
for (const k in derived[type][op])
|
|
||||||
{
|
|
||||||
sum_diff[type][op] = sum_diff[type][op] || {};
|
|
||||||
sum_diff[type][op][k] = (sum_diff[type][op][k] || 0n) + derived[type][op][k];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return sum_diff;
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum_object_counts(this.state, this.config)
|
|
||||||
function sum_object_counts(state, global_config)
|
|
||||||
{
|
|
||||||
const object_counts = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
|
|
||||||
const object_bytes = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
|
|
||||||
let pgstats = state.pgstats;
|
|
||||||
if (state.pg.stats)
|
|
||||||
{
|
|
||||||
// Merge with old stats for seamless transition to new stats
|
|
||||||
for (const pool_id in state.pg.stats)
|
|
||||||
{
|
|
||||||
pgstats[pool_id] = { ...(state.pg.stats[pool_id] || {}), ...(pgstats[pool_id] || {}) };
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const pool_id in pgstats)
|
|
||||||
{
|
|
||||||
let object_size = 0;
|
|
||||||
for (const osd_num of pgstats[pool_id].write_osd_set||[])
|
|
||||||
{
|
|
||||||
if (osd_num && state.osd.stats[osd_num] && state.osd.stats[osd_num].block_size)
|
|
||||||
{
|
|
||||||
object_size = state.osd.stats[osd_num].block_size;
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
const pool_cfg = (state.config.pools[pool_id]||{});
|
|
||||||
if (!object_size)
|
|
||||||
{
|
|
||||||
object_size = pool_cfg.block_size || global_config.block_size || 131072;
|
|
||||||
}
|
|
||||||
if (pool_cfg.scheme !== 'replicated')
|
|
||||||
{
|
|
||||||
object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0));
|
|
||||||
}
|
|
||||||
object_size = BigInt(object_size);
|
|
||||||
for (const pg_num in pgstats[pool_id])
|
|
||||||
{
|
|
||||||
const st = pgstats[pool_id][pg_num];
|
|
||||||
if (st)
|
|
||||||
{
|
|
||||||
for (const k in object_counts)
|
|
||||||
{
|
|
||||||
if (st[k+'_count'])
|
|
||||||
{
|
|
||||||
object_counts[k] += BigInt(st[k+'_count']);
|
|
||||||
object_bytes[k] += BigInt(st[k+'_count']) * object_size;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return { object_counts, object_bytes };
|
|
||||||
}
|
|
||||||
|
|
||||||
// sum_inode_stats(this.state, this.prev_stats)
|
|
||||||
function sum_inode_stats(state, prev_stats)
|
|
||||||
{
|
|
||||||
const inode_stats = {};
|
|
||||||
const inode_stub = () => ({
|
|
||||||
raw_used: 0n,
|
|
||||||
read: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
|
||||||
write: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
|
||||||
delete: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
|
||||||
});
|
|
||||||
const seen_pools = {};
|
|
||||||
for (const pool_id in state.config.pools)
|
|
||||||
{
|
|
||||||
seen_pools[pool_id] = true;
|
|
||||||
state.pool.stats[pool_id] = state.pool.stats[pool_id] || {};
|
|
||||||
state.pool.stats[pool_id].used_raw_tb = 0n;
|
|
||||||
}
|
|
||||||
for (const osd_num in state.osd.space)
|
|
||||||
{
|
|
||||||
for (const pool_id in state.osd.space[osd_num])
|
|
||||||
{
|
|
||||||
state.pool.stats[pool_id] = state.pool.stats[pool_id] || {};
|
|
||||||
if (!seen_pools[pool_id])
|
|
||||||
{
|
|
||||||
state.pool.stats[pool_id].used_raw_tb = 0n;
|
|
||||||
seen_pools[pool_id] = true;
|
|
||||||
}
|
|
||||||
inode_stats[pool_id] = inode_stats[pool_id] || {};
|
|
||||||
for (const inode_num in state.osd.space[osd_num][pool_id])
|
|
||||||
{
|
|
||||||
const u = BigInt(state.osd.space[osd_num][pool_id][inode_num]||0);
|
|
||||||
if (inode_num)
|
|
||||||
{
|
|
||||||
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
||||||
inode_stats[pool_id][inode_num].raw_used += u;
|
|
||||||
}
|
|
||||||
state.pool.stats[pool_id].used_raw_tb += u;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const pool_id in seen_pools)
|
|
||||||
{
|
|
||||||
const used = state.pool.stats[pool_id].used_raw_tb;
|
|
||||||
state.pool.stats[pool_id].used_raw_tb = Number(used)/1024/1024/1024/1024;
|
|
||||||
}
|
|
||||||
for (const osd_num in state.osd.state)
|
|
||||||
{
|
|
||||||
const ist = state.osd.inodestats[osd_num];
|
|
||||||
if (!ist || !state.osd.state[osd_num])
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for (const pool_id in ist)
|
|
||||||
{
|
|
||||||
inode_stats[pool_id] = inode_stats[pool_id] || {};
|
|
||||||
for (const inode_num in ist[pool_id])
|
|
||||||
{
|
|
||||||
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
||||||
for (const op of [ 'read', 'write', 'delete' ])
|
|
||||||
{
|
|
||||||
inode_stats[pool_id][inode_num][op].count += BigInt(ist[pool_id][inode_num][op].count||0);
|
|
||||||
inode_stats[pool_id][inode_num][op].usec += BigInt(ist[pool_id][inode_num][op].usec||0);
|
|
||||||
inode_stats[pool_id][inode_num][op].bytes += BigInt(ist[pool_id][inode_num][op].bytes||0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const osd in state.osd.state)
|
|
||||||
{
|
|
||||||
const osd_diff = prev_stats.osd_diff[osd];
|
|
||||||
if (!osd_diff || !state.osd.state[osd])
|
|
||||||
{
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
for (const pool_id in osd_diff.inode_stats)
|
|
||||||
{
|
|
||||||
for (const inode_num in prev_stats.osd_diff[osd].inode_stats[pool_id])
|
|
||||||
{
|
|
||||||
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
||||||
for (const op of [ 'read', 'write', 'delete' ])
|
|
||||||
{
|
|
||||||
const op_diff = prev_stats.osd_diff[osd].inode_stats[pool_id][inode_num][op] || {};
|
|
||||||
const op_st = inode_stats[pool_id][inode_num][op];
|
|
||||||
op_st.bps += op_diff.bps;
|
|
||||||
op_st.iops += op_diff.iops;
|
|
||||||
op_st.lat += op_diff.lat;
|
|
||||||
op_st.n_osd = (op_st.n_osd || 0) + 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
for (const pool_id in inode_stats)
|
|
||||||
{
|
|
||||||
for (const inode_num in inode_stats[pool_id])
|
|
||||||
{
|
|
||||||
let nonzero = inode_stats[pool_id][inode_num].raw_used > 0;
|
|
||||||
for (const op of [ 'read', 'write', 'delete' ])
|
|
||||||
{
|
|
||||||
const op_st = inode_stats[pool_id][inode_num][op];
|
|
||||||
if (op_st.n_osd)
|
|
||||||
{
|
|
||||||
op_st.lat /= BigInt(op_st.n_osd);
|
|
||||||
delete op_st.n_osd;
|
|
||||||
}
|
|
||||||
if (op_st.bps > 0 || op_st.iops > 0)
|
|
||||||
nonzero = true;
|
|
||||||
}
|
|
||||||
if (!nonzero && (!state.config.inode[pool_id] || !state.config.inode[pool_id][inode_num]))
|
|
||||||
{
|
|
||||||
// Deleted inode (no data, no I/O, no config)
|
|
||||||
delete inode_stats[pool_id][inode_num];
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return { inode_stats, seen_pools };
|
|
||||||
}
|
|
||||||
|
|
||||||
function serialize_bigints(obj)
|
|
||||||
{
|
|
||||||
obj = { ...obj };
|
|
||||||
for (const k in obj)
|
|
||||||
{
|
|
||||||
if (typeof obj[k] == 'bigint')
|
|
||||||
{
|
|
||||||
obj[k] = ''+obj[k];
|
|
||||||
}
|
|
||||||
else if (typeof obj[k] == 'object')
|
|
||||||
{
|
|
||||||
obj[k] = serialize_bigints(obj[k]);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return obj;
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
derive_osd_stats,
|
|
||||||
sum_op_stats,
|
|
||||||
sum_object_counts,
|
|
||||||
sum_inode_stats,
|
|
||||||
serialize_bigints,
|
|
||||||
};
|
|
|
@ -8,7 +8,7 @@
|
||||||
// But we support this case with the "parity_space" parameter in optimize_initial()/optimize_change().
|
// But we support this case with the "parity_space" parameter in optimize_initial()/optimize_change().
|
||||||
|
|
||||||
const { SimpleCombinator } = require('./simple_pgs.js');
|
const { SimpleCombinator } = require('./simple_pgs.js');
|
||||||
const LPOptimizer = require('./lp_optimizer.js');
|
const LPOptimizer = require('./lp-optimizer.js');
|
||||||
|
|
||||||
const osd_tree = {
|
const osd_tree = {
|
||||||
ripper5: {
|
ripper5: {
|
|
@ -2,7 +2,7 @@
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
const { compat } = require('./simple_pgs.js');
|
const { compat } = require('./simple_pgs.js');
|
||||||
const LPOptimizer = require('./lp_optimizer.js');
|
const LPOptimizer = require('./lp-optimizer.js');
|
||||||
|
|
||||||
async function run()
|
async function run()
|
||||||
{
|
{
|
|
@ -2,7 +2,7 @@
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
const { compat, flatten_tree } = require('./simple_pgs.js');
|
const { compat, flatten_tree } = require('./simple_pgs.js');
|
||||||
const LPOptimizer = require('./lp_optimizer.js');
|
const LPOptimizer = require('./lp-optimizer.js');
|
||||||
|
|
||||||
const crush_tree = [
|
const crush_tree = [
|
||||||
{ level: 1, children: [
|
{ level: 1, children: [
|
|
@ -2,7 +2,7 @@
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
const { compat } = require('./simple_pgs.js');
|
const { compat } = require('./simple_pgs.js');
|
||||||
const LPOptimizer = require('./lp_optimizer.js');
|
const LPOptimizer = require('./lp-optimizer.js');
|
||||||
|
|
||||||
const osd_tree = {
|
const osd_tree = {
|
||||||
100: {
|
100: {
|
|
@ -2,7 +2,7 @@
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
const { compat, flatten_tree } = require('./simple_pgs.js');
|
const { compat, flatten_tree } = require('./simple_pgs.js');
|
||||||
const LPOptimizer = require('./lp_optimizer.js');
|
const LPOptimizer = require('./lp-optimizer.js');
|
||||||
|
|
||||||
const osd_tree = {
|
const osd_tree = {
|
||||||
100: {
|
100: {
|
37
mon/utils.js
37
mon/utils.js
|
@ -1,37 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
const os = require('os');
|
|
||||||
|
|
||||||
function local_ips(all)
|
|
||||||
{
|
|
||||||
const ips = [];
|
|
||||||
const ifaces = os.networkInterfaces();
|
|
||||||
for (const ifname in ifaces)
|
|
||||||
{
|
|
||||||
for (const iface of ifaces[ifname])
|
|
||||||
{
|
|
||||||
if (iface.family == 'IPv4' && !iface.internal || all)
|
|
||||||
{
|
|
||||||
ips.push(iface.address);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return ips;
|
|
||||||
}
|
|
||||||
|
|
||||||
function b64(str)
|
|
||||||
{
|
|
||||||
return Buffer.from(str).toString('base64');
|
|
||||||
}
|
|
||||||
|
|
||||||
function de64(str)
|
|
||||||
{
|
|
||||||
return Buffer.from(str, 'base64').toString();
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = {
|
|
||||||
b64,
|
|
||||||
de64,
|
|
||||||
local_ips,
|
|
||||||
};
|
|
|
@ -1,49 +0,0 @@
|
||||||
// AntiEtcd persistence filter for Vitastor
|
|
||||||
// (c) Vitaliy Filippov, 2024
|
|
||||||
// License: Mozilla Public License 2.0 or Vitastor Network Public License 1.1
|
|
||||||
|
|
||||||
function vitastor_persist_filter(cfg)
|
|
||||||
{
|
|
||||||
const prefix = cfg.vitastor_prefix || '/vitastor';
|
|
||||||
return (key, value) =>
|
|
||||||
{
|
|
||||||
if (key.substr(0, prefix.length+'/osd/stats/'.length) == prefix+'/osd/stats/')
|
|
||||||
{
|
|
||||||
if (value)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
value = JSON.parse(value);
|
|
||||||
value = JSON.stringify({
|
|
||||||
bitmap_granularity: value.bitmap_granularity || undefined,
|
|
||||||
data_block_size: value.data_block_size || undefined,
|
|
||||||
host: value.host || undefined,
|
|
||||||
immediate_commit: value.immediate_commit || undefined,
|
|
||||||
});
|
|
||||||
}
|
|
||||||
catch (e)
|
|
||||||
{
|
|
||||||
console.error('invalid JSON in '+key+' = '+value+': '+e);
|
|
||||||
value = '{}';
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
value = undefined;
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
else if (key.substr(0, prefix.length+'/osd/'.length) == prefix+'/osd/' ||
|
|
||||||
key.substr(0, prefix.length+'/inode/stats/'.length) == prefix+'/inode/stats/' ||
|
|
||||||
key.substr(0, prefix.length+'/pg/stats/'.length) == prefix+'/pg/stats/' || // old name
|
|
||||||
key.substr(0, prefix.length+'/pgstats/'.length) == prefix+'/pgstats/' ||
|
|
||||||
key.substr(0, prefix.length+'/pool/stats/'.length) == prefix+'/pool/stats/' ||
|
|
||||||
key == prefix+'/stats')
|
|
||||||
{
|
|
||||||
return undefined;
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
module.exports = vitastor_persist_filter;
|
|
|
@ -1,80 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
#include "addon.h"
|
|
||||||
|
|
||||||
// Initialize the node addon
|
|
||||||
NAN_MODULE_INIT(InitAddon)
|
|
||||||
{
|
|
||||||
// vitastor.Client
|
|
||||||
|
|
||||||
v8::Local<v8::FunctionTemplate> tpl = Nan::New<v8::FunctionTemplate>(NodeVitastor::Create);
|
|
||||||
tpl->SetClassName(Nan::New("Client").ToLocalChecked());
|
|
||||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
|
||||||
|
|
||||||
Nan::SetPrototypeMethod(tpl, "read", NodeVitastor::Read);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "write", NodeVitastor::Write);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "sync", NodeVitastor::Sync);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "read_bitmap", NodeVitastor::ReadBitmap);
|
|
||||||
//Nan::SetPrototypeMethod(tpl, "destroy", NodeVitastor::Destroy);
|
|
||||||
|
|
||||||
Nan::Set(target, Nan::New("Client").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
|
|
||||||
|
|
||||||
// vitastor.Image (opened image)
|
|
||||||
|
|
||||||
tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorImage::Create);
|
|
||||||
tpl->SetClassName(Nan::New("Image").ToLocalChecked());
|
|
||||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
|
||||||
|
|
||||||
Nan::SetPrototypeMethod(tpl, "read", NodeVitastorImage::Read);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "write", NodeVitastorImage::Write);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "sync", NodeVitastorImage::Sync);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "get_info", NodeVitastorImage::GetInfo);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "read_bitmap", NodeVitastorImage::ReadBitmap);
|
|
||||||
|
|
||||||
Nan::Set(target, Nan::New("Image").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
|
|
||||||
|
|
||||||
// vitastor.KV
|
|
||||||
|
|
||||||
tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKV::Create);
|
|
||||||
tpl->SetClassName(Nan::New("KV").ToLocalChecked());
|
|
||||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
|
||||||
|
|
||||||
Nan::SetPrototypeMethod(tpl, "open", NodeVitastorKV::Open);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "set_config", NodeVitastorKV::SetConfig);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "close", NodeVitastorKV::Close);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "get_size", NodeVitastorKV::GetSize);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "get", NodeVitastorKV::Get);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "get_cached", NodeVitastorKV::GetCached);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "set", NodeVitastorKV::Set);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "del", NodeVitastorKV::Del);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "list", NodeVitastorKV::List);
|
|
||||||
|
|
||||||
Nan::Set(target, Nan::New("KV").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
|
|
||||||
|
|
||||||
Nan::Set(target, Nan::New("ENOENT").ToLocalChecked(), Nan::New<v8::Int32>(-ENOENT));
|
|
||||||
Nan::Set(target, Nan::New("EIO").ToLocalChecked(), Nan::New<v8::Int32>(-EIO));
|
|
||||||
Nan::Set(target, Nan::New("EINVAL").ToLocalChecked(), Nan::New<v8::Int32>(-EINVAL));
|
|
||||||
Nan::Set(target, Nan::New("EROFS").ToLocalChecked(), Nan::New<v8::Int32>(-EROFS));
|
|
||||||
Nan::Set(target, Nan::New("ENOSPC").ToLocalChecked(), Nan::New<v8::Int32>(-ENOSPC));
|
|
||||||
Nan::Set(target, Nan::New("EINTR").ToLocalChecked(), Nan::New<v8::Int32>(-EINTR));
|
|
||||||
Nan::Set(target, Nan::New("EILSEQ").ToLocalChecked(), Nan::New<v8::Int32>(-EILSEQ));
|
|
||||||
Nan::Set(target, Nan::New("ENOTBLK").ToLocalChecked(), Nan::New<v8::Int32>(-ENOTBLK));
|
|
||||||
Nan::Set(target, Nan::New("ENOSYS").ToLocalChecked(), Nan::New<v8::Int32>(-ENOSYS));
|
|
||||||
Nan::Set(target, Nan::New("EAGAIN").ToLocalChecked(), Nan::New<v8::Int32>(-EAGAIN));
|
|
||||||
|
|
||||||
// Listing handle
|
|
||||||
|
|
||||||
tpl = Nan::New<v8::FunctionTemplate>(NodeVitastorKVListing::Create);
|
|
||||||
tpl->SetClassName(Nan::New("KVListing").ToLocalChecked());
|
|
||||||
tpl->InstanceTemplate()->SetInternalFieldCount(1);
|
|
||||||
|
|
||||||
Nan::SetPrototypeMethod(tpl, "next", NodeVitastorKVListing::Next);
|
|
||||||
Nan::SetPrototypeMethod(tpl, "close", NodeVitastorKVListing::Close);
|
|
||||||
|
|
||||||
Nan::Set(target, Nan::New("KVListing").ToLocalChecked(), Nan::GetFunction(tpl).ToLocalChecked());
|
|
||||||
|
|
||||||
NodeVitastorKV::listing_class.Reset(Nan::GetFunction(tpl).ToLocalChecked());
|
|
||||||
}
|
|
||||||
|
|
||||||
NODE_MODULE(addon, (void*)InitAddon)
|
|
|
@ -1,20 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
#ifndef NODE_VITASTOR_ADDON_H
|
|
||||||
#define NODE_VITASTOR_ADDON_H
|
|
||||||
|
|
||||||
#include <nan.h>
|
|
||||||
#include <vitastor_c.h>
|
|
||||||
|
|
||||||
#include "client.h"
|
|
||||||
|
|
||||||
#define ERRORF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
|
|
||||||
|
|
||||||
//#define TRACEF(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
|
|
||||||
//#define TRACE(msg) fprintf(stderr, "%s\n", msg);
|
|
||||||
|
|
||||||
#define TRACEF(format, ...) ;
|
|
||||||
#define TRACE(msg) ;
|
|
||||||
|
|
||||||
#endif
|
|
|
@ -1,21 +0,0 @@
|
||||||
{
|
|
||||||
'targets': [
|
|
||||||
{
|
|
||||||
'target_name': 'addon',
|
|
||||||
'sources': [
|
|
||||||
'client.cc',
|
|
||||||
'addon.cc'
|
|
||||||
],
|
|
||||||
'include_dirs': [
|
|
||||||
'<!(node -e "require(\'nan\')")'
|
|
||||||
],
|
|
||||||
'cflags': [
|
|
||||||
'<!(pkg-config --cflags vitastor)'
|
|
||||||
],
|
|
||||||
'libraries': [
|
|
||||||
'<!(pkg-config --libs vitastor)',
|
|
||||||
'-lvitastor_kv'
|
|
||||||
]
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
|
@ -1,981 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
#include "addon.h"
|
|
||||||
|
|
||||||
#define NODE_VITASTOR_READ 1
|
|
||||||
#define NODE_VITASTOR_WRITE 2
|
|
||||||
#define NODE_VITASTOR_SYNC 3
|
|
||||||
#define NODE_VITASTOR_READ_BITMAP 4
|
|
||||||
#define NODE_VITASTOR_GET_INFO 5
|
|
||||||
|
|
||||||
#ifndef INODE_POOL
|
|
||||||
#define INODE_POOL(inode) (uint32_t)((inode) >> (64 - POOL_ID_BITS))
|
|
||||||
#define INODE_NO_POOL(inode) (uint64_t)((inode) & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1))
|
|
||||||
#define INODE_WITH_POOL(pool_id, inode) (((uint64_t)(pool_id) << (64-POOL_ID_BITS)) | INODE_NO_POOL(inode))
|
|
||||||
#endif
|
|
||||||
|
|
||||||
class NodeVitastorRequest: public Nan::AsyncResource
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
NodeVitastorRequest(NodeVitastor *cli, v8::Local<v8::Function> cb): Nan::AsyncResource("NodeVitastorRequest")
|
|
||||||
{
|
|
||||||
this->cli = cli;
|
|
||||||
callback.Reset(cb);
|
|
||||||
}
|
|
||||||
|
|
||||||
iovec iov;
|
|
||||||
std::vector<iovec> iov_list;
|
|
||||||
NodeVitastor *cli = NULL;
|
|
||||||
NodeVitastorImage *img = NULL;
|
|
||||||
int op = 0;
|
|
||||||
uint64_t offset = 0, len = 0, version = 0;
|
|
||||||
bool with_parents = false;
|
|
||||||
Nan::Persistent<v8::Function> callback;
|
|
||||||
};
|
|
||||||
|
|
||||||
static uint64_t get_ui64(const v8::Local<v8::Value> & val)
|
|
||||||
{
|
|
||||||
if (val->IsBigInt())
|
|
||||||
return val->ToBigInt(Nan::GetCurrentContext()).ToLocalChecked()->Uint64Value();
|
|
||||||
return Nan::To<int64_t>(val).FromJust();
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
// NodeVitastor
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
|
|
||||||
NodeVitastor::NodeVitastor(): Nan::ObjectWrap()
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor: constructor");
|
|
||||||
poll_watcher.data = this;
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastor::~NodeVitastor()
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor: destructor");
|
|
||||||
uv_poll_stop(&poll_watcher);
|
|
||||||
vitastor_c_destroy(c);
|
|
||||||
c = NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
NAN_METHOD(NodeVitastor::Create)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::Create");
|
|
||||||
v8::Local<v8::Object> jsParams = info[0].As<v8::Object>();
|
|
||||||
v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
|
|
||||||
std::vector<std::string> cfg;
|
|
||||||
for (uint32_t i = 0; i < keys->Length(); i++)
|
|
||||||
{
|
|
||||||
auto key = Nan::Get(keys, i).ToLocalChecked();
|
|
||||||
cfg.push_back(std::string(*Nan::Utf8String(key)));
|
|
||||||
cfg.push_back(std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked())));
|
|
||||||
}
|
|
||||||
|
|
||||||
const char **c_cfg = new const char*[cfg.size()];
|
|
||||||
for (size_t i = 0; i < cfg.size(); i++)
|
|
||||||
{
|
|
||||||
c_cfg[i] = cfg[i].c_str();
|
|
||||||
}
|
|
||||||
NodeVitastor* cli = new NodeVitastor();
|
|
||||||
cli->c = vitastor_c_create_uring_json(c_cfg, cfg.size());
|
|
||||||
delete[] c_cfg;
|
|
||||||
|
|
||||||
int res = vitastor_c_uring_register_eventfd(cli->c);
|
|
||||||
if (res >= 0)
|
|
||||||
{
|
|
||||||
cli->eventfd = res;
|
|
||||||
res = uv_poll_init(uv_default_loop(), &cli->poll_watcher, cli->eventfd);
|
|
||||||
if (res >= 0)
|
|
||||||
res = uv_poll_start(&cli->poll_watcher, UV_READABLE, on_io_readable);
|
|
||||||
on_io_readable(&cli->poll_watcher, 0, UV_READABLE);
|
|
||||||
}
|
|
||||||
if (res < 0)
|
|
||||||
{
|
|
||||||
ERRORF("NodeVitastor: failed to create and register io_uring eventfd in libuv: %s", strerror(-cli->eventfd));
|
|
||||||
vitastor_c_destroy(cli->c);
|
|
||||||
cli->c = NULL;
|
|
||||||
Nan::ThrowError("failed to create and register io_uring eventfd");
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
cli->Wrap(info.This());
|
|
||||||
info.GetReturnValue().Set(info.This());
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastor::on_io_readable(uv_poll_t* handle, int status, int revents)
|
|
||||||
{
|
|
||||||
TRACEF("NodeVitastor::on_io_readable status/revents %d %d", status, revents);
|
|
||||||
if (revents & UV_READABLE)
|
|
||||||
{
|
|
||||||
NodeVitastor* self = (NodeVitastor*)handle->data;
|
|
||||||
vitastor_c_uring_handle_events(self->c);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorRequest* NodeVitastor::get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
|
|
||||||
{
|
|
||||||
uint64_t offset = get_ui64(info[argpos+0]);
|
|
||||||
uint64_t len = get_ui64(info[argpos+1]);
|
|
||||||
uint8_t *buf = (uint8_t*)malloc(len);
|
|
||||||
if (!buf)
|
|
||||||
{
|
|
||||||
Nan::ThrowError("failed to allocate memory");
|
|
||||||
return NULL;
|
|
||||||
}
|
|
||||||
v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(this, callback);
|
|
||||||
|
|
||||||
req->offset = offset;
|
|
||||||
req->len = len;
|
|
||||||
req->iov = { .iov_base = buf, .iov_len = len };
|
|
||||||
|
|
||||||
return req;
|
|
||||||
}
|
|
||||||
|
|
||||||
// read(pool, inode, offset, len, callback(err, buffer, version))
|
|
||||||
NAN_METHOD(NodeVitastor::Read)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::Read");
|
|
||||||
if (info.Length() < 5)
|
|
||||||
Nan::ThrowError("Not enough arguments to read(pool, inode, offset, len, callback(err, buffer, version))");
|
|
||||||
|
|
||||||
NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
|
|
||||||
|
|
||||||
uint64_t pool = get_ui64(info[0]);
|
|
||||||
uint64_t inode = get_ui64(info[1]);
|
|
||||||
|
|
||||||
auto req = self->get_read_request(info, 2);
|
|
||||||
|
|
||||||
self->Ref();
|
|
||||||
vitastor_c_read(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, &req->iov, 1, on_read_finish, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorRequest* NodeVitastor::get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos)
|
|
||||||
{
|
|
||||||
uint64_t offset = get_ui64(info[argpos+0]);
|
|
||||||
const auto & bufarg = info[argpos+1];
|
|
||||||
uint64_t version = 0;
|
|
||||||
if (!info[argpos+2].IsEmpty() &&
|
|
||||||
!info[argpos+2]->IsFunction() &&
|
|
||||||
info[argpos+2]->IsObject())
|
|
||||||
{
|
|
||||||
auto key = Nan::New<v8::String>("version").ToLocalChecked();
|
|
||||||
auto params = info[argpos+2].As<v8::Object>();
|
|
||||||
auto versionObj = Nan::Get(params, key).ToLocalChecked();
|
|
||||||
if (!versionObj.IsEmpty())
|
|
||||||
version = get_ui64(versionObj);
|
|
||||||
argpos++;
|
|
||||||
}
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[argpos+2].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(this, callback);
|
|
||||||
|
|
||||||
req->offset = offset;
|
|
||||||
req->version = version;
|
|
||||||
|
|
||||||
if (bufarg->IsArray())
|
|
||||||
{
|
|
||||||
auto buffers = bufarg.As<v8::Array>();
|
|
||||||
req->len = 0;
|
|
||||||
for (uint32_t i = 0; i < buffers->Length(); i++)
|
|
||||||
{
|
|
||||||
auto buffer_obj = Nan::Get(buffers, i).ToLocalChecked();
|
|
||||||
char *buf = node::Buffer::Data(buffer_obj);
|
|
||||||
uint64_t len = node::Buffer::Length(buffer_obj);
|
|
||||||
req->iov_list.push_back({ .iov_base = buf, .iov_len = len });
|
|
||||||
req->len += len;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
char *buf = node::Buffer::Data(bufarg);
|
|
||||||
uint64_t len = node::Buffer::Length(bufarg);
|
|
||||||
req->iov = { .iov_base = buf, .iov_len = len };
|
|
||||||
req->len = len;
|
|
||||||
}
|
|
||||||
|
|
||||||
return req;
|
|
||||||
}
|
|
||||||
|
|
||||||
// write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
|
|
||||||
NAN_METHOD(NodeVitastor::Write)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::Write");
|
|
||||||
if (info.Length() < 5)
|
|
||||||
Nan::ThrowError("Not enough arguments to write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))");
|
|
||||||
|
|
||||||
NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
|
|
||||||
|
|
||||||
uint64_t pool = get_ui64(info[0]);
|
|
||||||
uint64_t inode = get_ui64(info[1]);
|
|
||||||
|
|
||||||
auto req = self->get_write_request(info, 2);
|
|
||||||
|
|
||||||
self->Ref();
|
|
||||||
vitastor_c_write(self->c, ((pool << (64-POOL_ID_BITS)) | inode), req->offset, req->len, req->version,
|
|
||||||
req->iov_list.size() ? req->iov_list.data() : &req->iov,
|
|
||||||
req->iov_list.size() ? req->iov_list.size() : 1,
|
|
||||||
on_write_finish, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// sync(callback(err))
|
|
||||||
NAN_METHOD(NodeVitastor::Sync)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::Sync");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to sync(callback(err))");
|
|
||||||
|
|
||||||
NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[0].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(self, callback);
|
|
||||||
|
|
||||||
self->Ref();
|
|
||||||
vitastor_c_sync(self->c, on_write_finish, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
|
|
||||||
NAN_METHOD(NodeVitastor::ReadBitmap)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::ReadBitmap");
|
|
||||||
if (info.Length() < 6)
|
|
||||||
Nan::ThrowError("Not enough arguments to read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))");
|
|
||||||
|
|
||||||
NodeVitastor* self = Nan::ObjectWrap::Unwrap<NodeVitastor>(info.This());
|
|
||||||
|
|
||||||
uint64_t pool = get_ui64(info[0]);
|
|
||||||
uint64_t inode = get_ui64(info[1]);
|
|
||||||
uint64_t offset = get_ui64(info[2]);
|
|
||||||
uint64_t len = get_ui64(info[3]);
|
|
||||||
bool with_parents = Nan::To<bool>(info[4]).FromJust();
|
|
||||||
v8::Local<v8::Function> callback = info[5].As<v8::Function>();
|
|
||||||
|
|
||||||
auto req = new NodeVitastorRequest(self, callback);
|
|
||||||
self->Ref();
|
|
||||||
vitastor_c_read_bitmap(self->c, ((pool << (64-POOL_ID_BITS)) | inode), offset, len, with_parents, on_read_bitmap_finish, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
static void on_error(NodeVitastorRequest *req, Nan::Callback & nanCallback, long retval)
|
|
||||||
{
|
|
||||||
// Legal errors: EINVAL, EIO, EROFS, ENOSPC, EINTR, ENOENT
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
if (!retval)
|
|
||||||
args[0] = Nan::Null();
|
|
||||||
else
|
|
||||||
args[0] = Nan::New<v8::Int32>((int32_t)retval);
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastor::on_read_finish(void *opaque, long retval, uint64_t version)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::on_read_finish");
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
if (retval == -ENOENT)
|
|
||||||
{
|
|
||||||
free(req->iov.iov_base);
|
|
||||||
nanCallback.Call(0, NULL, req);
|
|
||||||
}
|
|
||||||
else if (retval < 0 || (uint64_t)retval != req->len)
|
|
||||||
{
|
|
||||||
free(req->iov.iov_base);
|
|
||||||
on_error(req, nanCallback, retval);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
v8::Local<v8::Value> args[3];
|
|
||||||
args[0] = Nan::Null();
|
|
||||||
args[1] = Nan::NewBuffer((char*)req->iov.iov_base, req->iov.iov_len).ToLocalChecked();
|
|
||||||
args[2] = v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), version);
|
|
||||||
nanCallback.Call(3, args, req);
|
|
||||||
}
|
|
||||||
req->cli->Unref();
|
|
||||||
delete req;
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastor::on_write_finish(void *opaque, long retval)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::on_write_finish");
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
if (retval < 0 || (uint64_t)retval != req->len)
|
|
||||||
{
|
|
||||||
on_error(req, nanCallback, retval);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = Nan::Null();
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
}
|
|
||||||
req->cli->Unref();
|
|
||||||
delete req;
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastor::on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastor::on_read_bitmap_finish");
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
NodeVitastorRequest *req = (NodeVitastorRequest *)opaque;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
if (retval == -ENOENT)
|
|
||||||
nanCallback.Call(0, NULL, req);
|
|
||||||
else if (retval < 0)
|
|
||||||
on_error(req, nanCallback, retval);
|
|
||||||
else
|
|
||||||
{
|
|
||||||
v8::Local<v8::Value> args[2];
|
|
||||||
args[0] = Nan::Null();
|
|
||||||
args[1] = Nan::NewBuffer((char*)bitmap, (retval+7)/8).ToLocalChecked();
|
|
||||||
nanCallback.Call(2, args, req);
|
|
||||||
}
|
|
||||||
req->cli->Unref();
|
|
||||||
delete req;
|
|
||||||
}
|
|
||||||
|
|
||||||
//NAN_METHOD(NodeVitastor::Destroy)
|
|
||||||
//{
|
|
||||||
// TRACE("NodeVitastor::Destroy");
|
|
||||||
//}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
// NodeVitastorImage
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
|
|
||||||
NAN_METHOD(NodeVitastorImage::Create)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::Create");
|
|
||||||
if (info.Length() < 2)
|
|
||||||
Nan::ThrowError("Not enough arguments to Image(client, name)");
|
|
||||||
|
|
||||||
v8::Local<v8::Object> parent = info[0].As<v8::Object>();
|
|
||||||
std::string name = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
|
|
||||||
NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
|
|
||||||
|
|
||||||
NodeVitastorImage *img = new NodeVitastorImage();
|
|
||||||
img->Wrap(info.This());
|
|
||||||
|
|
||||||
img->cli = cli;
|
|
||||||
img->name = name;
|
|
||||||
|
|
||||||
img->Ref();
|
|
||||||
cli->Ref();
|
|
||||||
vitastor_c_watch_inode(cli->c, (char*)img->name.c_str(), on_watch_start, img);
|
|
||||||
|
|
||||||
info.GetReturnValue().Set(info.This());
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorImage::~NodeVitastorImage()
|
|
||||||
{
|
|
||||||
if (watch)
|
|
||||||
{
|
|
||||||
vitastor_c_close_watch(cli->c, watch);
|
|
||||||
watch = NULL;
|
|
||||||
}
|
|
||||||
cli->Unref();
|
|
||||||
}
|
|
||||||
|
|
||||||
// read(offset, len, callback(err, buffer, version))
|
|
||||||
NAN_METHOD(NodeVitastorImage::Read)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::Read");
|
|
||||||
if (info.Length() < 3)
|
|
||||||
Nan::ThrowError("Not enough arguments to read(offset, len, callback(err, buffer, version))");
|
|
||||||
|
|
||||||
NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
|
|
||||||
|
|
||||||
auto req = img->cli->get_read_request(info, 0);
|
|
||||||
req->img = img;
|
|
||||||
req->op = NODE_VITASTOR_READ;
|
|
||||||
|
|
||||||
img->exec_or_wait(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// write(offset, buffer, { version }?, callback(err))
|
|
||||||
NAN_METHOD(NodeVitastorImage::Write)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::Write");
|
|
||||||
if (info.Length() < 3)
|
|
||||||
Nan::ThrowError("Not enough arguments to write(offset, buffer, { version }?, callback(err))");
|
|
||||||
|
|
||||||
NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
|
|
||||||
|
|
||||||
auto req = img->cli->get_write_request(info, 0);
|
|
||||||
req->img = img;
|
|
||||||
req->op = NODE_VITASTOR_WRITE;
|
|
||||||
|
|
||||||
img->exec_or_wait(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// sync(callback(err))
|
|
||||||
NAN_METHOD(NodeVitastorImage::Sync)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::Sync");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to sync(callback(err))");
|
|
||||||
|
|
||||||
NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[0].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(img->cli, callback);
|
|
||||||
req->img = img;
|
|
||||||
req->op = NODE_VITASTOR_SYNC;
|
|
||||||
|
|
||||||
img->exec_or_wait(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
|
|
||||||
NAN_METHOD(NodeVitastorImage::ReadBitmap)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::ReadBitmap");
|
|
||||||
if (info.Length() < 4)
|
|
||||||
Nan::ThrowError("Not enough arguments to read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))");
|
|
||||||
|
|
||||||
NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
|
|
||||||
|
|
||||||
uint64_t offset = get_ui64(info[0]);
|
|
||||||
uint64_t len = get_ui64(info[1]);
|
|
||||||
bool with_parents = Nan::To<bool>(info[2]).FromJust();
|
|
||||||
v8::Local<v8::Function> callback = info[3].As<v8::Function>();
|
|
||||||
|
|
||||||
auto req = new NodeVitastorRequest(img->cli, callback);
|
|
||||||
req->img = img;
|
|
||||||
req->op = NODE_VITASTOR_READ_BITMAP;
|
|
||||||
req->offset = offset;
|
|
||||||
req->len = len;
|
|
||||||
req->with_parents = with_parents;
|
|
||||||
|
|
||||||
img->exec_or_wait(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
|
|
||||||
NAN_METHOD(NodeVitastorImage::GetInfo)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorImage::GetInfo");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))");
|
|
||||||
|
|
||||||
NodeVitastorImage* img = Nan::ObjectWrap::Unwrap<NodeVitastorImage>(info.This());
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[0].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(img->cli, callback);
|
|
||||||
req->img = img;
|
|
||||||
req->op = NODE_VITASTOR_GET_INFO;
|
|
||||||
|
|
||||||
img->exec_or_wait(req);
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastorImage::exec_or_wait(NodeVitastorRequest *req)
|
|
||||||
{
|
|
||||||
if (!watch)
|
|
||||||
{
|
|
||||||
// Need to wait for initialisation
|
|
||||||
on_init.push_back(req);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
exec_request(req);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastorImage::exec_request(NodeVitastorRequest *req)
|
|
||||||
{
|
|
||||||
if (req->op == NODE_VITASTOR_READ)
|
|
||||||
{
|
|
||||||
uint64_t ino = vitastor_c_inode_get_num(watch);
|
|
||||||
cli->Ref();
|
|
||||||
vitastor_c_read(cli->c, ino, req->offset, req->len, &req->iov, 1, NodeVitastor::on_read_finish, req);
|
|
||||||
}
|
|
||||||
else if (req->op == NODE_VITASTOR_WRITE)
|
|
||||||
{
|
|
||||||
uint64_t ino = vitastor_c_inode_get_num(watch);
|
|
||||||
cli->Ref();
|
|
||||||
vitastor_c_write(cli->c, ino, req->offset, req->len, req->version,
|
|
||||||
req->iov_list.size() ? req->iov_list.data() : &req->iov,
|
|
||||||
req->iov_list.size() ? req->iov_list.size() : 1,
|
|
||||||
NodeVitastor::on_write_finish, req);
|
|
||||||
}
|
|
||||||
else if (req->op == NODE_VITASTOR_SYNC)
|
|
||||||
{
|
|
||||||
uint64_t ino = vitastor_c_inode_get_num(watch);
|
|
||||||
uint32_t imm = vitastor_c_inode_get_immediate_commit(cli->c, ino);
|
|
||||||
cli->Ref();
|
|
||||||
if (imm != IMMEDIATE_ALL)
|
|
||||||
{
|
|
||||||
vitastor_c_sync(cli->c, NodeVitastor::on_write_finish, req);
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
NodeVitastor::on_write_finish(req, 0);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
else if (req->op == NODE_VITASTOR_READ_BITMAP)
|
|
||||||
{
|
|
||||||
uint64_t ino = vitastor_c_inode_get_num(watch);
|
|
||||||
cli->Ref();
|
|
||||||
vitastor_c_read_bitmap(cli->c, ino, req->offset, req->len, req->with_parents, NodeVitastor::on_read_bitmap_finish, req);
|
|
||||||
}
|
|
||||||
else if (req->op == NODE_VITASTOR_GET_INFO)
|
|
||||||
{
|
|
||||||
uint64_t size = vitastor_c_inode_get_size(watch);
|
|
||||||
uint64_t num = vitastor_c_inode_get_num(watch);
|
|
||||||
uint32_t block_size = vitastor_c_inode_get_block_size(cli->c, num);
|
|
||||||
uint32_t bitmap_granularity = vitastor_c_inode_get_bitmap_granularity(cli->c, num);
|
|
||||||
int readonly = vitastor_c_inode_get_readonly(watch);
|
|
||||||
uint32_t immediate_commit = vitastor_c_inode_get_immediate_commit(cli->c, num);
|
|
||||||
uint64_t parent_id = vitastor_c_inode_get_parent_id(watch);
|
|
||||||
char *meta = vitastor_c_inode_get_meta(watch);
|
|
||||||
uint64_t mod_revision = vitastor_c_inode_get_mod_revision(watch);
|
|
||||||
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
|
|
||||||
v8::Local<v8::Object> res = Nan::New<v8::Object>();
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(num)));
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(num)));
|
|
||||||
if (size < ((uint64_t)1<<53))
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), Nan::New<v8::Number>(size));
|
|
||||||
else
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("size").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), size));
|
|
||||||
if (parent_id)
|
|
||||||
{
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("parent_pool_id").ToLocalChecked(), Nan::New<v8::Number>(INODE_POOL(parent_id)));
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("parent_inode_num").ToLocalChecked(), Nan::New<v8::Number>(INODE_NO_POOL(parent_id)));
|
|
||||||
}
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("readonly").ToLocalChecked(), Nan::New((bool)readonly));
|
|
||||||
if (meta)
|
|
||||||
{
|
|
||||||
Nan::JSON nanJSON;
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("meta").ToLocalChecked(), nanJSON.Parse(Nan::New<v8::String>(meta).ToLocalChecked()).ToLocalChecked());
|
|
||||||
}
|
|
||||||
if (mod_revision < ((uint64_t)1<<53))
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), Nan::New<v8::Number>(mod_revision));
|
|
||||||
else
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("mod_revision").ToLocalChecked(), v8::BigInt::NewFromUnsigned(v8::Isolate::GetCurrent(), mod_revision));
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("block_size").ToLocalChecked(), Nan::New(block_size));
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("bitmap_granularity").ToLocalChecked(), Nan::New(bitmap_granularity));
|
|
||||||
Nan::Set(res, Nan::New<v8::String>("immediate_commit").ToLocalChecked(), Nan::New(immediate_commit));
|
|
||||||
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = res;
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
|
|
||||||
delete req;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastorImage::on_watch_start(void *opaque, long retval)
|
|
||||||
{
|
|
||||||
NodeVitastorImage *img = (NodeVitastorImage *)opaque;
|
|
||||||
{
|
|
||||||
img->watch = (void*)retval;
|
|
||||||
auto on_init = std::move(img->on_init);
|
|
||||||
for (auto req: on_init)
|
|
||||||
{
|
|
||||||
img->exec_request(req);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
img->Unref();
|
|
||||||
}
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
// NodeVitastorKV
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
|
|
||||||
Nan::Persistent<v8::Function> NodeVitastorKV::listing_class;
|
|
||||||
|
|
||||||
// constructor(node_vitastor)
|
|
||||||
NAN_METHOD(NodeVitastorKV::Create)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Create");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to new KV(client)");
|
|
||||||
|
|
||||||
v8::Local<v8::Object> parent = info[0].As<v8::Object>();
|
|
||||||
NodeVitastor *cli = Nan::ObjectWrap::Unwrap<NodeVitastor>(parent);
|
|
||||||
|
|
||||||
NodeVitastorKV *kv = new NodeVitastorKV();
|
|
||||||
kv->cli = cli;
|
|
||||||
{
|
|
||||||
kv->dbw = new vitastorkv_dbw_t((cluster_client_t*)vitastor_c_get_internal_client(cli->c));
|
|
||||||
}
|
|
||||||
|
|
||||||
kv->Wrap(info.This());
|
|
||||||
cli->Ref();
|
|
||||||
info.GetReturnValue().Set(info.This());
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorKV::~NodeVitastorKV()
|
|
||||||
{
|
|
||||||
delete dbw;
|
|
||||||
cli->Unref();
|
|
||||||
}
|
|
||||||
|
|
||||||
// open(pool_id, inode_num, { ...config }, callback(err))
|
|
||||||
NAN_METHOD(NodeVitastorKV::Open)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Open");
|
|
||||||
if (info.Length() < 4)
|
|
||||||
Nan::ThrowError("Not enough arguments to open(pool_id, inode_num, { ...config }, callback(err))");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
uint64_t inode_id = INODE_WITH_POOL(get_ui64(info[0]), get_ui64(info[1]));
|
|
||||||
|
|
||||||
v8::Local<v8::Object> jsParams = info[2].As<v8::Object>();
|
|
||||||
v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
|
|
||||||
std::map<std::string, std::string> cfg;
|
|
||||||
for (uint32_t i = 0; i < keys->Length(); i++)
|
|
||||||
{
|
|
||||||
auto key = Nan::Get(keys, i).ToLocalChecked();
|
|
||||||
cfg[std::string(*Nan::Utf8String(key))] = std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked()));
|
|
||||||
}
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[3].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(kv->cli, callback);
|
|
||||||
|
|
||||||
kv->Ref();
|
|
||||||
kv->dbw->open(inode_id, cfg, [kv, req](int res)
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = !res ? v8::Local<v8::Value>(Nan::Null()) : v8::Local<v8::Value>(Nan::New<v8::Int32>(res));
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
delete req;
|
|
||||||
kv->Unref();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// close(callback(err))
|
|
||||||
NAN_METHOD(NodeVitastorKV::Close)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Close");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to close(callback(err))");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[0].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(kv->cli, callback);
|
|
||||||
|
|
||||||
kv->Ref();
|
|
||||||
kv->dbw->close([kv, req]()
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
nanCallback.Call(0, NULL, req);
|
|
||||||
delete req;
|
|
||||||
kv->Unref();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// set_config({ ...config })
|
|
||||||
NAN_METHOD(NodeVitastorKV::SetConfig)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::SetConfig");
|
|
||||||
if (info.Length() < 1)
|
|
||||||
Nan::ThrowError("Not enough arguments to set_config({ ...config })");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
v8::Local<v8::Object> jsParams = info[0].As<v8::Object>();
|
|
||||||
v8::Local<v8::Array> keys = Nan::GetOwnPropertyNames(jsParams).ToLocalChecked();
|
|
||||||
std::map<std::string, std::string> cfg;
|
|
||||||
for (uint32_t i = 0; i < keys->Length(); i++)
|
|
||||||
{
|
|
||||||
auto key = Nan::Get(keys, i).ToLocalChecked();
|
|
||||||
cfg[std::string(*Nan::Utf8String(key))] = std::string(*Nan::Utf8String(Nan::Get(jsParams, key).ToLocalChecked()));
|
|
||||||
}
|
|
||||||
|
|
||||||
kv->dbw->set_config(cfg);
|
|
||||||
}
|
|
||||||
|
|
||||||
// get_size()
|
|
||||||
NAN_METHOD(NodeVitastorKV::GetSize)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::GetSize");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
auto size = kv->dbw->get_size();
|
|
||||||
info.GetReturnValue().Set((size < ((uint64_t)1<<53))
|
|
||||||
? v8::Local<v8::Value>(Nan::New<v8::Number>(size))
|
|
||||||
: v8::Local<v8::Value>(v8::BigInt::NewFromUnsigned(info.GetIsolate(), size)));
|
|
||||||
}
|
|
||||||
|
|
||||||
void NodeVitastorKV::get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info, bool allow_cache)
|
|
||||||
{
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
// FIXME: Handle Buffer too
|
|
||||||
std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[1].As<v8::Function>();
|
|
||||||
auto req = new NodeVitastorRequest(kv->cli, callback);
|
|
||||||
|
|
||||||
kv->Ref();
|
|
||||||
kv->dbw->get(key, [kv, req](int res, const std::string & value)
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[2];
|
|
||||||
args[0] = !res ? v8::Local<v8::Value>(Nan::Null()) : v8::Local<v8::Value>(Nan::New<v8::Int32>(res));
|
|
||||||
args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
|
|
||||||
nanCallback.Call(2, args, req);
|
|
||||||
delete req;
|
|
||||||
kv->Unref();
|
|
||||||
}, allow_cache);
|
|
||||||
}
|
|
||||||
|
|
||||||
// get(key, callback(err, value))
|
|
||||||
NAN_METHOD(NodeVitastorKV::Get)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Get");
|
|
||||||
if (info.Length() < 2)
|
|
||||||
Nan::ThrowError("Not enough arguments to get(key, callback(err, value))");
|
|
||||||
get_impl(info, false);
|
|
||||||
}
|
|
||||||
|
|
||||||
// get_cached(key, callback(err, value))
|
|
||||||
NAN_METHOD(NodeVitastorKV::GetCached)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::GetCached");
|
|
||||||
if (info.Length() < 2)
|
|
||||||
Nan::ThrowError("Not enough arguments to get_cached(key, callback(err, value))");
|
|
||||||
get_impl(info, true);
|
|
||||||
}
|
|
||||||
|
|
||||||
static std::function<bool(int, const std::string &)> make_cas_callback(NodeVitastorRequest *cas_req)
|
|
||||||
{
|
|
||||||
return [cas_req](int res, const std::string & value)
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(cas_req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
|
|
||||||
Nan::MaybeLocal<v8::Value> ret = nanCallback.Call(1, args, cas_req);
|
|
||||||
if (ret.IsEmpty())
|
|
||||||
return false;
|
|
||||||
return Nan::To<bool>(ret.ToLocalChecked()).FromJust();
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
// set(key, value, callback(err), cas_compare(old_value)?)
|
|
||||||
NAN_METHOD(NodeVitastorKV::Set)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Set");
|
|
||||||
if (info.Length() < 3)
|
|
||||||
Nan::ThrowError("Not enough arguments to set(key, value, callback(err), cas_compare(old_value)?)");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
// FIXME: Handle Buffer too
|
|
||||||
std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
|
|
||||||
std::string value(*Nan::Utf8String(info[1].As<v8::String>()));
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[2].As<v8::Function>();
|
|
||||||
NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
|
|
||||||
|
|
||||||
std::function<bool(int, const std::string &)> cas_cb;
|
|
||||||
if (info.Length() > 3 && info[3]->IsObject())
|
|
||||||
{
|
|
||||||
v8::Local<v8::Function> cas_callback = info[3].As<v8::Function>();
|
|
||||||
cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
|
|
||||||
cas_cb = make_cas_callback(cas_req);
|
|
||||||
}
|
|
||||||
|
|
||||||
kv->Ref();
|
|
||||||
kv->dbw->set(key, value, [kv, req, cas_req](int res)
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = !res ? v8::Local<v8::Value>(Nan::Null()) : v8::Local<v8::Value>(Nan::New<v8::Int32>(res));
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
delete req;
|
|
||||||
if (cas_req)
|
|
||||||
delete cas_req;
|
|
||||||
kv->Unref();
|
|
||||||
}, cas_cb);
|
|
||||||
}
|
|
||||||
|
|
||||||
// del(key, callback(err), cas_compare(old_value)?)
|
|
||||||
NAN_METHOD(NodeVitastorKV::Del)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Del");
|
|
||||||
if (info.Length() < 2)
|
|
||||||
Nan::ThrowError("Not enough arguments to del(key, callback(err), cas_compare(old_value)?)");
|
|
||||||
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
|
|
||||||
// FIXME: Handle Buffer too
|
|
||||||
std::string key(*Nan::Utf8String(info[0].As<v8::String>()));
|
|
||||||
|
|
||||||
v8::Local<v8::Function> callback = info[1].As<v8::Function>();
|
|
||||||
NodeVitastorRequest *req = new NodeVitastorRequest(kv->cli, callback), *cas_req = NULL;
|
|
||||||
|
|
||||||
std::function<bool(int, const std::string &)> cas_cb;
|
|
||||||
if (info.Length() > 2 && info[2]->IsObject())
|
|
||||||
{
|
|
||||||
v8::Local<v8::Function> cas_callback = info[2].As<v8::Function>();
|
|
||||||
cas_req = new NodeVitastorRequest(kv->cli, cas_callback);
|
|
||||||
cas_cb = make_cas_callback(cas_req);
|
|
||||||
}
|
|
||||||
|
|
||||||
kv->Ref();
|
|
||||||
kv->dbw->del(key, [kv, req, cas_req](int res)
|
|
||||||
{
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = !res ? v8::Local<v8::Value>(Nan::Null()) : v8::Local<v8::Value>(Nan::New<v8::Int32>(res));
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
delete req;
|
|
||||||
if (cas_req)
|
|
||||||
delete cas_req;
|
|
||||||
kv->Unref();
|
|
||||||
}, cas_cb);
|
|
||||||
}
|
|
||||||
|
|
||||||
// list(start_key?)
|
|
||||||
NAN_METHOD(NodeVitastorKV::List)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::List");
|
|
||||||
|
|
||||||
v8::Local<v8::Function> cons = Nan::New(listing_class);
|
|
||||||
v8::Local<v8::Value> args[2];
|
|
||||||
args[0] = info.This();
|
|
||||||
int narg = 1;
|
|
||||||
if (info.Length() > 1 && info[1]->IsString())
|
|
||||||
{
|
|
||||||
args[1] = info[1];
|
|
||||||
narg = 2;
|
|
||||||
}
|
|
||||||
info.GetReturnValue().Set(Nan::NewInstance(cons, narg, args).ToLocalChecked());
|
|
||||||
}
|
|
||||||
|
|
||||||
/*NAN_METHOD(NodeVitastorKV::Destroy)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKV::Destroy");
|
|
||||||
NodeVitastorKV* kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(info.This());
|
|
||||||
if (!kv->dead)
|
|
||||||
kv->Unref();
|
|
||||||
}*/
|
|
||||||
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
// NodeVitastorKVListing
|
|
||||||
//////////////////////////////////////////////////
|
|
||||||
|
|
||||||
// constructor(node_vitastor_kv, start_key?)
|
|
||||||
NAN_METHOD(NodeVitastorKVListing::Create)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKVListing::Create");
|
|
||||||
|
|
||||||
v8::Local<v8::Object> parent = info[0].As<v8::Object>();
|
|
||||||
NodeVitastorKV *kv = Nan::ObjectWrap::Unwrap<NodeVitastorKV>(parent);
|
|
||||||
|
|
||||||
std::string start_key;
|
|
||||||
// FIXME: Handle Buffer too
|
|
||||||
if (info.Length() > 1 && info[1]->IsString())
|
|
||||||
{
|
|
||||||
start_key = std::string(*Nan::Utf8String(info[1].As<v8::String>()));
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorKVListing *list = new NodeVitastorKVListing();
|
|
||||||
list->kv = kv;
|
|
||||||
list->handle = list->kv->dbw->list_start(start_key);
|
|
||||||
|
|
||||||
list->Wrap(info.This());
|
|
||||||
kv->Ref();
|
|
||||||
info.GetReturnValue().Set(info.This());
|
|
||||||
}
|
|
||||||
|
|
||||||
NodeVitastorKVListing::~NodeVitastorKVListing()
|
|
||||||
{
|
|
||||||
if (handle)
|
|
||||||
{
|
|
||||||
kv->dbw->list_close(handle);
|
|
||||||
handle = NULL;
|
|
||||||
}
|
|
||||||
if (iter)
|
|
||||||
{
|
|
||||||
delete iter;
|
|
||||||
iter = NULL;
|
|
||||||
}
|
|
||||||
kv->Unref();
|
|
||||||
}
|
|
||||||
|
|
||||||
// next(callback(err, value)?)
|
|
||||||
NAN_METHOD(NodeVitastorKVListing::Next)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKVListing::Next");
|
|
||||||
|
|
||||||
NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
|
|
||||||
|
|
||||||
if (info.Length() > 0)
|
|
||||||
{
|
|
||||||
v8::Local<v8::Function> callback = info[0].As<v8::Function>();
|
|
||||||
if (list->iter)
|
|
||||||
{
|
|
||||||
delete list->iter;
|
|
||||||
}
|
|
||||||
list->iter = new NodeVitastorRequest(list->kv->cli, callback);
|
|
||||||
}
|
|
||||||
if (!list->handle)
|
|
||||||
{
|
|
||||||
// Already closed
|
|
||||||
if (list->iter)
|
|
||||||
{
|
|
||||||
auto req = list->iter;
|
|
||||||
list->iter = NULL;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[1];
|
|
||||||
args[0] = Nan::New<v8::Int32>(-EINVAL);
|
|
||||||
nanCallback.Call(1, args, req);
|
|
||||||
delete req;
|
|
||||||
}
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
list->kv->Ref();
|
|
||||||
list->kv->dbw->list_next(list->handle, [list](int res, const std::string & key, const std::string & value)
|
|
||||||
{
|
|
||||||
auto req = list->iter;
|
|
||||||
list->iter = NULL;
|
|
||||||
Nan::HandleScope scope;
|
|
||||||
Nan::Callback nanCallback(Nan::New(req->callback));
|
|
||||||
v8::Local<v8::Value> args[3];
|
|
||||||
args[0] = Nan::New<v8::Int32>(res);
|
|
||||||
args[1] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(key).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
|
|
||||||
args[2] = !res ? v8::Local<v8::Value>(Nan::New<v8::String>(value).ToLocalChecked()) : v8::Local<v8::Value>(Nan::Null());
|
|
||||||
nanCallback.Call(3, args, req);
|
|
||||||
if (list->iter)
|
|
||||||
delete req;
|
|
||||||
else
|
|
||||||
list->iter = req;
|
|
||||||
list->kv->Unref();
|
|
||||||
});
|
|
||||||
}
|
|
||||||
|
|
||||||
// close()
|
|
||||||
NAN_METHOD(NodeVitastorKVListing::Close)
|
|
||||||
{
|
|
||||||
TRACE("NodeVitastorKVListing::Close");
|
|
||||||
|
|
||||||
NodeVitastorKVListing* list = Nan::ObjectWrap::Unwrap<NodeVitastorKVListing>(info.This());
|
|
||||||
|
|
||||||
if (list->handle)
|
|
||||||
{
|
|
||||||
list->kv->dbw->list_close(list->handle);
|
|
||||||
list->handle = NULL;
|
|
||||||
}
|
|
||||||
if (list->iter)
|
|
||||||
{
|
|
||||||
delete list->iter;
|
|
||||||
list->iter = NULL;
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,136 +0,0 @@
|
||||||
// Copyright (c) Vitaliy Filippov, 2019+
|
|
||||||
// License: VNPL-1.1 (see README.md for details)
|
|
||||||
|
|
||||||
#ifndef NODE_VITASTOR_CLIENT_H
|
|
||||||
#define NODE_VITASTOR_CLIENT_H
|
|
||||||
|
|
||||||
#include <nan.h>
|
|
||||||
#include <vitastor_c.h>
|
|
||||||
#include <vitastor_kv.h>
|
|
||||||
|
|
||||||
class NodeVitastorRequest;
|
|
||||||
|
|
||||||
class NodeVitastor: public Nan::ObjectWrap
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// constructor({ ...config })
|
|
||||||
static NAN_METHOD(Create);
|
|
||||||
// read(pool, inode, offset, len, callback(err, buffer, version))
|
|
||||||
static NAN_METHOD(Read);
|
|
||||||
// write(pool, inode, offset, buf: Buffer | Buffer[], { version }?, callback(err))
|
|
||||||
static NAN_METHOD(Write);
|
|
||||||
// sync(callback(err))
|
|
||||||
static NAN_METHOD(Sync);
|
|
||||||
// read_bitmap(pool, inode, offset, len, with_parents, callback(err, bitmap_buffer))
|
|
||||||
static NAN_METHOD(ReadBitmap);
|
|
||||||
// // destroy()
|
|
||||||
// static NAN_METHOD(Destroy);
|
|
||||||
|
|
||||||
~NodeVitastor();
|
|
||||||
|
|
||||||
private:
|
|
||||||
vitastor_c *c = NULL;
|
|
||||||
int eventfd = -1;
|
|
||||||
uv_poll_t poll_watcher;
|
|
||||||
|
|
||||||
NodeVitastor();
|
|
||||||
|
|
||||||
static void on_io_readable(uv_poll_t* handle, int status, int revents);
|
|
||||||
static void on_read_finish(void *opaque, long retval, uint64_t version);
|
|
||||||
static void on_write_finish(void *opaque, long retval);
|
|
||||||
static void on_read_bitmap_finish(void *opaque, long retval, uint8_t *bitmap);
|
|
||||||
|
|
||||||
NodeVitastorRequest* get_read_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
|
|
||||||
NodeVitastorRequest* get_write_request(const Nan::FunctionCallbackInfo<v8::Value> & info, int argpos);
|
|
||||||
|
|
||||||
friend class NodeVitastorImage;
|
|
||||||
friend class NodeVitastorKV;
|
|
||||||
friend class NodeVitastorKVListing;
|
|
||||||
};
|
|
||||||
|
|
||||||
class NodeVitastorImage: public Nan::ObjectWrap
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// constructor(node_vitastor, name)
|
|
||||||
static NAN_METHOD(Create);
|
|
||||||
// read(offset, len, callback(err, buffer, version))
|
|
||||||
static NAN_METHOD(Read);
|
|
||||||
// write(offset, buf: Buffer | Buffer[], { version }?, callback(err))
|
|
||||||
static NAN_METHOD(Write);
|
|
||||||
// sync(callback(err))
|
|
||||||
static NAN_METHOD(Sync);
|
|
||||||
// read_bitmap(offset, len, with_parents, callback(err, bitmap_buffer))
|
|
||||||
static NAN_METHOD(ReadBitmap);
|
|
||||||
// get_info(callback({ num, name, size, parent_id?, readonly?, meta?, mod_revision, block_size, bitmap_granularity, immediate_commit }))
|
|
||||||
static NAN_METHOD(GetInfo);
|
|
||||||
|
|
||||||
~NodeVitastorImage();
|
|
||||||
|
|
||||||
private:
|
|
||||||
NodeVitastor *cli = NULL;
|
|
||||||
std::string name;
|
|
||||||
void *watch = NULL;
|
|
||||||
std::vector<NodeVitastorRequest*> on_init;
|
|
||||||
Nan::Persistent<v8::Object> cliObj;
|
|
||||||
|
|
||||||
static void on_watch_start(void *opaque, long retval);
|
|
||||||
void exec_request(NodeVitastorRequest *req);
|
|
||||||
void exec_or_wait(NodeVitastorRequest *req);
|
|
||||||
};
|
|
||||||
|
|
||||||
class NodeVitastorKV: public Nan::ObjectWrap
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// constructor(node_vitastor)
|
|
||||||
static NAN_METHOD(Create);
|
|
||||||
// open(pool_id, inode_num, { ...config }, callback(err))
|
|
||||||
static NAN_METHOD(Open);
|
|
||||||
// set_config({ ...config })
|
|
||||||
static NAN_METHOD(SetConfig);
|
|
||||||
// close(callback())
|
|
||||||
static NAN_METHOD(Close);
|
|
||||||
// get_size()
|
|
||||||
static NAN_METHOD(GetSize);
|
|
||||||
// get(key, callback(err, value))
|
|
||||||
static NAN_METHOD(Get);
|
|
||||||
// get_cached(key, callback(err, value))
|
|
||||||
static NAN_METHOD(GetCached);
|
|
||||||
// set(key, value, callback(err), cas_compare(old_value)?)
|
|
||||||
static NAN_METHOD(Set);
|
|
||||||
// del(key, callback(err), cas_compare(old_value)?)
|
|
||||||
static NAN_METHOD(Del);
|
|
||||||
// list(start_key?)
|
|
||||||
static NAN_METHOD(List);
|
|
||||||
|
|
||||||
~NodeVitastorKV();
|
|
||||||
|
|
||||||
static Nan::Persistent<v8::Function> listing_class;
|
|
||||||
|
|
||||||
private:
|
|
||||||
NodeVitastor *cli = NULL;
|
|
||||||
vitastorkv_dbw_t *dbw = NULL;
|
|
||||||
|
|
||||||
static void get_impl(const Nan::FunctionCallbackInfo<v8::Value> & info, bool allow_cache);
|
|
||||||
|
|
||||||
friend class NodeVitastorKVListing;
|
|
||||||
};
|
|
||||||
|
|
||||||
class NodeVitastorKVListing: public Nan::ObjectWrap
|
|
||||||
{
|
|
||||||
public:
|
|
||||||
// constructor(node_vitastor_kv, start_key?)
|
|
||||||
static NAN_METHOD(Create);
|
|
||||||
// next(callback(err, value)?)
|
|
||||||
static NAN_METHOD(Next);
|
|
||||||
// close()
|
|
||||||
static NAN_METHOD(Close);
|
|
||||||
|
|
||||||
~NodeVitastorKVListing();
|
|
||||||
|
|
||||||
private:
|
|
||||||
NodeVitastorKV *kv = NULL;
|
|
||||||
void *handle = NULL;
|
|
||||||
NodeVitastorRequest *iter = NULL;
|
|
||||||
};
|
|
||||||
|
|
||||||
#endif
|
|
|
@ -1 +0,0 @@
|
||||||
module.exports = require('bindings')('addon.node');
|
|
|
@ -1,24 +0,0 @@
|
||||||
{
|
|
||||||
"name": "vitastor",
|
|
||||||
"version": "1.7.0",
|
|
||||||
"description": "Low-level native bindings to Vitastor client library",
|
|
||||||
"main": "index.js",
|
|
||||||
"keywords": [
|
|
||||||
"storage",
|
|
||||||
"sds",
|
|
||||||
"vitastor"
|
|
||||||
],
|
|
||||||
"repository": {
|
|
||||||
"type": "git",
|
|
||||||
"url": "git://git.yourcmc.ru/vitalif/vitastor.git"
|
|
||||||
},
|
|
||||||
"scripts": {
|
|
||||||
"build": "node-gyp rebuild"
|
|
||||||
},
|
|
||||||
"author": "Vitaliy Filippov",
|
|
||||||
"license": "VNPL-2.0",
|
|
||||||
"dependencies": {
|
|
||||||
"bindings": "1.5.0",
|
|
||||||
"nan": "^2.19.0"
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1 +0,0 @@
|
||||||
See [../docs/installation/opennebula.en.md](../docs/installation/opennebula.en.md).
|
|
|
@ -1,29 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
reapply_patch() {
|
|
||||||
if ! patch -f --dry-run -F 0 -R $1 < $2 >/dev/null; then
|
|
||||||
already_applied=0
|
|
||||||
if ! patch --no-backup-if-mismatch -r - -F 0 -f $1 < $2; then
|
|
||||||
applied_ok=0
|
|
||||||
echo "ERROR: Failed to patch file $1, please apply the patch $2 manually"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
echo "Reapplying Vitastor patches to OpenNebula's oned.conf, vmm_execrc and downloader.sh"
|
|
||||||
already_applied=1
|
|
||||||
applied_ok=1
|
|
||||||
reapply_patch /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/vitastor/downloader-vitastor.sh.diff
|
|
||||||
reapply_patch /etc/one/oned.conf /var/lib/one/remotes/datastore/vitastor/oned.conf.diff
|
|
||||||
reapply_patch /etc/one/vmm_exec/vmm_execrc /var/lib/one/remotes/datastore/vitastor/vmm_execrc.diff
|
|
||||||
if [[ "$already_applied" = 1 ]]; then
|
|
||||||
echo "OK: Vitastor OpenNebula patches are already applied"
|
|
||||||
elif [[ "$applied_ok" = 1 ]]; then
|
|
||||||
echo "OK: Vitastor OpenNebula patches successfully applied"
|
|
||||||
fi
|
|
||||||
if [[ -f /etc/apparmor.d/local/abstractions/libvirt-qemu ]]; then
|
|
||||||
if ! grep -q /etc/vitastor/vitastor.conf /etc/apparmor.d/local/abstractions/libvirt-qemu; then
|
|
||||||
echo ' "/etc/vitastor/vitastor.conf" r,' >> /etc/apparmor.d/local/abstractions/libvirt-qemu
|
|
||||||
fi
|
|
||||||
fi
|
|
|
@ -1,76 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Vitastor OpenNebula driver
|
|
||||||
# Copyright (c) Vitaliy Filippov, 2024+
|
|
||||||
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
# This script is used to copy a VM image (SRC) to the image repository as DST
|
|
||||||
|
|
||||||
# -------- Set up the environment to source common tools & conf ------------
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
source ${DRIVER_PATH}/../libfs.sh
|
|
||||||
|
|
||||||
# -------- Get cp and datastore arguments from OpenNebula core ------------
|
|
||||||
|
|
||||||
DRV_ACTION=`cat -`
|
|
||||||
ID=$1
|
|
||||||
|
|
||||||
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
|
|
||||||
|
|
||||||
unset i
|
|
||||||
|
|
||||||
BASE_PATH="${XPATH_ELEMENTS[i++]}"
|
|
||||||
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
|
|
||||||
POOL_NAME="${XPATH_ELEMENTS[i++]}"
|
|
||||||
IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
|
|
||||||
SRC="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SIZE="${XPATH_ELEMENTS[i++]}"
|
|
||||||
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
|
|
||||||
|
|
||||||
DST_HOST=`get_destination_host $ID`
|
|
||||||
|
|
||||||
if [ -z "$DST_HOST" ]; then
|
|
||||||
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLI=vitastor-cli
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path ${VITASTOR_CONF}"
|
|
||||||
fi
|
|
||||||
if [ -n "$POOL_NAME" ]; then
|
|
||||||
CLI="$CLI --pool ${POOL_NAME}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
SAFE_DIRS=""
|
|
||||||
|
|
||||||
DST="${IMAGE_PREFIX}-${ID}"
|
|
||||||
|
|
||||||
ssh_exec_and_log "$DST_HOST" "$CLI create --parent $SRC $DST" \
|
|
||||||
"Error during $CLI create --parent $SRC $DST in $DST_HOST"
|
|
||||||
|
|
||||||
ssh_exec_and_log "$DST_HOST" "$CLI flatten $DST" \
|
|
||||||
"Error during $CLI create flatten $DST in $DST_HOST"
|
|
||||||
|
|
||||||
echo "$DST raw"
|
|
|
@ -1,135 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Vitastor OpenNebula driver
|
|
||||||
# Copyright (c) Vitaliy Filippov, 2024+
|
|
||||||
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
# This script is used to copy a local image SRC to the image repository as DST
|
|
||||||
|
|
||||||
# -------- Set up the environment to source common tools & conf ------------
|
|
||||||
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
source ${DRIVER_PATH}/../libfs.sh
|
|
||||||
|
|
||||||
# -------- Get cp and datastore arguments from OpenNebula core ------------
|
|
||||||
|
|
||||||
DRV_ACTION=`cat -`
|
|
||||||
ID=$1
|
|
||||||
|
|
||||||
export DRV_ACTION
|
|
||||||
|
|
||||||
UTILS_PATH="${DRIVER_PATH}/.."
|
|
||||||
|
|
||||||
XPATH="$UTILS_PATH/xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/STAGING_DIR \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/PATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/NO_DECOMPRESS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/LIMIT_TRANSFER_BW \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
|
|
||||||
|
|
||||||
unset i
|
|
||||||
|
|
||||||
BASE_PATH="${XPATH_ELEMENTS[i++]}"
|
|
||||||
RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SAFE_DIRS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
|
|
||||||
POOL_NAME="${XPATH_ELEMENTS[i++]}"
|
|
||||||
IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
|
|
||||||
STAGING_DIR="${XPATH_ELEMENTS[i++]:-/var/tmp}"
|
|
||||||
SRC="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SIZE="${XPATH_ELEMENTS[i++]}"
|
|
||||||
MD5="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SHA1="${XPATH_ELEMENTS[i++]}"
|
|
||||||
NO_DECOMPRESS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
LIMIT_TRANSFER_BW="${XPATH_ELEMENTS[i++]}"
|
|
||||||
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
|
|
||||||
|
|
||||||
DST_HOST=`get_destination_host $ID`
|
|
||||||
|
|
||||||
if [ -z "$DST_HOST" ]; then
|
|
||||||
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLI=vitastor-cli
|
|
||||||
QEMU_ARG=""
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path ${VITASTOR_CONF}"
|
|
||||||
QEMU_ARG=":config_path=${VITASTOR_CONF}"
|
|
||||||
fi
|
|
||||||
if [ -n "$POOL_NAME" ]; then
|
|
||||||
CLI="$CLI --pool ${POOL_NAME}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS"
|
|
||||||
|
|
||||||
IMAGE_HASH=`generate_image_hash`
|
|
||||||
TMP_DST="$STAGING_DIR/$IMAGE_HASH"
|
|
||||||
|
|
||||||
DST="${IMAGE_PREFIX}-${ID}"
|
|
||||||
|
|
||||||
DOWNLOADER_ARGS=`set_downloader_args "$MD5" "$SHA1" "$NO_DECOMPRESS" "$LIMIT_TRANSFER_BW" "$SRC" -`
|
|
||||||
|
|
||||||
COPY_COMMAND="$UTILS_PATH/downloader.sh $DOWNLOADER_ARGS"
|
|
||||||
|
|
||||||
case $SRC in
|
|
||||||
http://*|https://*)
|
|
||||||
log "Downloading $SRC to the image repository"
|
|
||||||
|
|
||||||
DUMP="$COPY_COMMAND"
|
|
||||||
;;
|
|
||||||
|
|
||||||
*)
|
|
||||||
if [ `check_restricted $SRC` -eq 1 ]; then
|
|
||||||
log_error "Not allowed to copy images from $RESTRICTED_DIRS"
|
|
||||||
error_message "Not allowed to copy image file $SRC"
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Copying local image $SRC to the image repository"
|
|
||||||
|
|
||||||
DUMP="$COPY_COMMAND"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
multiline_exec_and_log "set -e -o pipefail; $DUMP | $SSH $DST_HOST $DD of=$TMP_DST bs=1M" \
|
|
||||||
"Error copying $SRC to $DST_HOST:$TMP_DST"
|
|
||||||
|
|
||||||
REGISTER_CMD=$(cat <<EOF
|
|
||||||
set -e -o pipefail
|
|
||||||
|
|
||||||
SIZE=\$($QEMU_IMG info --output json "$TMP_DST" | jq -r '.["virtual-size"]')
|
|
||||||
$CLI create -s \$SIZE "$DST"
|
|
||||||
$QEMU_IMG convert -O raw "$TMP_DST" "vitastor:image=$DST$QEMU_ARG"
|
|
||||||
|
|
||||||
# remove original
|
|
||||||
$RM -f $TMP_DST
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
ssh_exec_and_log "$DST_HOST" "$REGISTER_CMD" "Error registering $DST in $DST_HOST"
|
|
||||||
|
|
||||||
echo "$DST raw"
|
|
|
@ -1,555 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# -------------------------------------------------------------------------- #
|
|
||||||
# Copyright 2002-2023, OpenNebula Project, OpenNebula Systems #
|
|
||||||
# #
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may #
|
|
||||||
# not use this file except in compliance with the License. You may obtain #
|
|
||||||
# a copy of the License at #
|
|
||||||
# #
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0 #
|
|
||||||
# #
|
|
||||||
# Unless required by applicable law or agreed to in writing, software #
|
|
||||||
# distributed under the License is distributed on an "AS IS" BASIS, #
|
|
||||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. #
|
|
||||||
# See the License for the specific language governing permissions and #
|
|
||||||
# limitations under the License. #
|
|
||||||
#--------------------------------------------------------------------------- #
|
|
||||||
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
VAR_LOCATION=/var/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
VAR_LOCATION=$ONE_LOCATION/var
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
|
|
||||||
# Escape single quotes
|
|
||||||
function esc_sq
|
|
||||||
{
|
|
||||||
echo "$1" | sed -e "s/'/'\\\''/g"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Execute a command (first parameter) and use the first kb of stdout
|
|
||||||
# to determine the file type
|
|
||||||
function get_type
|
|
||||||
{
|
|
||||||
if [ "$NO_DECOMPRESS" = "yes" ]; then
|
|
||||||
echo "application/octet-stream"
|
|
||||||
else
|
|
||||||
command=$1
|
|
||||||
|
|
||||||
( eval "$command" | head -n 1024 | file -b --mime-type - ) 2>/dev/null
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Gets the command needed to decompress an stream.
|
|
||||||
function get_decompressor
|
|
||||||
{
|
|
||||||
type=$1
|
|
||||||
|
|
||||||
case "$type" in
|
|
||||||
"application/x-gzip"|"application/gzip")
|
|
||||||
echo "gunzip -c -"
|
|
||||||
;;
|
|
||||||
"application/x-bzip2")
|
|
||||||
echo "bunzip2 -qc -"
|
|
||||||
;;
|
|
||||||
"application/x-xz")
|
|
||||||
echo "unxz -c -"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
echo "cat"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
}
|
|
||||||
|
|
||||||
# Function called to decompress a stream. The first parameter is the command
|
|
||||||
# used to decompress the stream. Second parameter is the output file or
|
|
||||||
# - for stdout.
|
|
||||||
function decompress
|
|
||||||
{
|
|
||||||
command="$1"
|
|
||||||
to="$2"
|
|
||||||
|
|
||||||
if [ "$to" = "-" ]; then
|
|
||||||
$command
|
|
||||||
else
|
|
||||||
$command > "$to"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Function called to hash a stream. First parameter is the algorithm name.
|
|
||||||
function hasher
|
|
||||||
{
|
|
||||||
if [ -n "$1" ]; then
|
|
||||||
openssl dgst -$1 | awk '{print $NF}' > $HASH_FILE
|
|
||||||
else
|
|
||||||
# Needs something consuming stdin or the pipe will break
|
|
||||||
cat >/dev/null
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
# Unarchives a tar or a zip a file to a directory with the same name.
|
|
||||||
function unarchive
|
|
||||||
{
|
|
||||||
TO="$1"
|
|
||||||
|
|
||||||
file_type=$(get_type "cat $TO")
|
|
||||||
|
|
||||||
tmp="$TO"
|
|
||||||
|
|
||||||
# Add full path if it is relative
|
|
||||||
if [ ${tmp:0:1} != "/" ]; then
|
|
||||||
tmp="$PWD/$tmp"
|
|
||||||
fi
|
|
||||||
|
|
||||||
IN="$tmp.tmp"
|
|
||||||
OUT="$tmp"
|
|
||||||
|
|
||||||
case "$file_type" in
|
|
||||||
"application/x-tar")
|
|
||||||
command="tar -xf $IN -C $OUT"
|
|
||||||
;;
|
|
||||||
"application/zip")
|
|
||||||
command="unzip -d $OUT $IN"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
command=""
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [ -n "$command" ]; then
|
|
||||||
mv "$OUT" "$IN"
|
|
||||||
mkdir "$OUT"
|
|
||||||
|
|
||||||
$command
|
|
||||||
|
|
||||||
if [ "$?" != "0" ]; then
|
|
||||||
echo "Error uncompressing archive" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
rm "$IN"
|
|
||||||
fi
|
|
||||||
}
|
|
||||||
|
|
||||||
function s3_env
|
|
||||||
{
|
|
||||||
XPATH="$DRIVER_PATH/xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i j XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH /DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ACCESS_KEY_ID \
|
|
||||||
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/SECRET_ACCESS_KEY \
|
|
||||||
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/REGION \
|
|
||||||
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/AWS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/MARKETPLACE/TEMPLATE/ENDPOINT)
|
|
||||||
|
|
||||||
S3_ACCESS_KEY_ID="${XPATH_ELEMENTS[j++]}"
|
|
||||||
S3_SECRET_ACCESS_KEY="${XPATH_ELEMENTS[j++]}"
|
|
||||||
S3_REGION="${XPATH_ELEMENTS[j++]}"
|
|
||||||
S3_AWS="${XPATH_ELEMENTS[j++]}"
|
|
||||||
S3_ENDPOINT="${XPATH_ELEMENTS[j++]}"
|
|
||||||
|
|
||||||
CURRENT_DATE_DAY="$(date -u '+%Y%m%d')"
|
|
||||||
CURRENT_DATE_ISO8601="${CURRENT_DATE_DAY}T$(date -u '+%H%M%S')Z"
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create an SHA-256 hash in hexadecimal.
|
|
||||||
# Usage:
|
|
||||||
# hash_sha256 <string>
|
|
||||||
function hash_sha256 {
|
|
||||||
printf "${1}" | openssl dgst -sha256 | sed 's/^.* //'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create an SHA-256 hmac in hexadecimal.
|
|
||||||
# Usage:
|
|
||||||
# hmac_sha256 <key> <data>
|
|
||||||
function hmac_sha256 {
|
|
||||||
printf "${2}" | openssl dgst -sha256 -mac HMAC -macopt "${1}" | sed 's/^.* //'
|
|
||||||
}
|
|
||||||
|
|
||||||
# Create the signature.
|
|
||||||
# Usage:
|
|
||||||
# create_signature
|
|
||||||
function create_signature {
|
|
||||||
stringToSign="AWS4-HMAC-SHA256\n${CURRENT_DATE_ISO8601}\n${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request\n$(hash_sha256 "${HTTP_CANONICAL_REQUEST}")"
|
|
||||||
dateKey=$(hmac_sha256 key:"AWS4${S3_SECRET_ACCESS_KEY}" "${CURRENT_DATE_DAY}")
|
|
||||||
regionKey=$(hmac_sha256 hexkey:"${dateKey}" "${S3_REGION}")
|
|
||||||
serviceKey=$(hmac_sha256 hexkey:"${regionKey}" "s3")
|
|
||||||
signingKey=$(hmac_sha256 hexkey:"${serviceKey}" "aws4_request")
|
|
||||||
|
|
||||||
printf "${stringToSign}" | openssl dgst -sha256 -mac HMAC -macopt hexkey:"${signingKey}" | sed 's/.*(stdin)= //'
|
|
||||||
}
|
|
||||||
|
|
||||||
function s3_curl_args
|
|
||||||
{
|
|
||||||
FROM="$1"
|
|
||||||
|
|
||||||
ENDPOINT="$S3_ENDPOINT"
|
|
||||||
OBJECT=$(basename "$FROM")
|
|
||||||
BUCKET=$(basename $(dirname "$FROM"))
|
|
||||||
|
|
||||||
DATE="`date -u +'%a, %d %b %Y %H:%M:%S GMT'`"
|
|
||||||
AUTH_STRING="GET\n\n\n${DATE}\n/${BUCKET}/${OBJECT}"
|
|
||||||
|
|
||||||
SIGNED_AUTH_STRING=`echo -en "$AUTH_STRING" | \
|
|
||||||
openssl sha1 -hmac ${S3_SECRET_ACCESS_KEY} -binary | \
|
|
||||||
base64`
|
|
||||||
|
|
||||||
echo " -H \"Date: ${DATE}\"" \
|
|
||||||
" -H \"Authorization: AWS ${S3_ACCESS_KEY_ID}:${SIGNED_AUTH_STRING}\"" \
|
|
||||||
" '$(esc_sq "${ENDPOINT}/${BUCKET}/${OBJECT}")'"
|
|
||||||
}
|
|
||||||
|
|
||||||
function s3_curl_args_aws
|
|
||||||
{
|
|
||||||
FROM="$1"
|
|
||||||
|
|
||||||
OBJECT=$(basename "$FROM")
|
|
||||||
BUCKET=$(basename "$(dirname "$FROM")")
|
|
||||||
|
|
||||||
ENDPOINT="$BUCKET.s3.amazonaws.com"
|
|
||||||
|
|
||||||
AWS_S3_PATH="$(echo $OBJECT | sed 's;^\([^/]\);/\1;')"
|
|
||||||
|
|
||||||
HTTP_REQUEST_PAYLOAD_HASH="$(echo "" | openssl dgst -sha256 | sed 's/^.* //')"
|
|
||||||
HTTP_CANONICAL_REQUEST_URI="${AWS_S3_PATH}"
|
|
||||||
HTTP_REQUEST_CONTENT_TYPE='application/octet-stream'
|
|
||||||
|
|
||||||
HTTP_CANONICAL_REQUEST_HEADERS="content-type:${HTTP_REQUEST_CONTENT_TYPE}
|
|
||||||
host:${ENDPOINT}
|
|
||||||
x-amz-content-sha256:${HTTP_REQUEST_PAYLOAD_HASH}
|
|
||||||
x-amz-date:${CURRENT_DATE_ISO8601}"
|
|
||||||
|
|
||||||
HTTP_REQUEST_SIGNED_HEADERS="content-type;host;x-amz-content-sha256;x-amz-date"
|
|
||||||
HTTP_CANONICAL_REQUEST="GET
|
|
||||||
${HTTP_CANONICAL_REQUEST_URI}\n
|
|
||||||
${HTTP_CANONICAL_REQUEST_HEADERS}\n
|
|
||||||
${HTTP_REQUEST_SIGNED_HEADERS}
|
|
||||||
${HTTP_REQUEST_PAYLOAD_HASH}"
|
|
||||||
|
|
||||||
SIGNATURE="$(create_signature)"
|
|
||||||
HTTP_REQUEST_AUTHORIZATION_HEADER="AWS4-HMAC-SHA256 Credential=${S3_ACCESS_KEY_ID}/${CURRENT_DATE_DAY}/${S3_REGION}/s3/aws4_request, SignedHeaders=${HTTP_REQUEST_SIGNED_HEADERS}, Signature=${SIGNATURE}"
|
|
||||||
|
|
||||||
echo " -H \"Authorization: ${HTTP_REQUEST_AUTHORIZATION_HEADER}\"" \
|
|
||||||
" -H \"content-type: ${HTTP_REQUEST_CONTENT_TYPE}\"" \
|
|
||||||
" -H \"x-amz-content-sha256: ${HTTP_REQUEST_PAYLOAD_HASH}\"" \
|
|
||||||
" -H \"x-amz-date: ${CURRENT_DATE_ISO8601}\"" \
|
|
||||||
" \"https://${ENDPOINT}${HTTP_CANONICAL_REQUEST_URI}\""
|
|
||||||
}
|
|
||||||
|
|
||||||
function get_rbd_cmd
|
|
||||||
{
|
|
||||||
local i j URL_ELEMENTS
|
|
||||||
|
|
||||||
FROM="$1"
|
|
||||||
|
|
||||||
URL_RB="$DRIVER_PATH/url.rb"
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
URL_ELEMENTS[i++]="$element"
|
|
||||||
done < <($URL_RB "$FROM" \
|
|
||||||
USER \
|
|
||||||
HOST \
|
|
||||||
SOURCE \
|
|
||||||
PARAM_DS \
|
|
||||||
PARAM_CEPH_USER \
|
|
||||||
PARAM_CEPH_KEY \
|
|
||||||
PARAM_CEPH_CONF)
|
|
||||||
|
|
||||||
USER="${URL_ELEMENTS[j++]}"
|
|
||||||
DST_HOST="${URL_ELEMENTS[j++]}"
|
|
||||||
SOURCE="${URL_ELEMENTS[j++]}"
|
|
||||||
DS="${URL_ELEMENTS[j++]}"
|
|
||||||
CEPH_USER="${URL_ELEMENTS[j++]}"
|
|
||||||
CEPH_KEY="${URL_ELEMENTS[j++]}"
|
|
||||||
CEPH_CONF="${URL_ELEMENTS[j++]}"
|
|
||||||
|
|
||||||
# Remove leading '/'
|
|
||||||
SOURCE="${SOURCE#/}"
|
|
||||||
|
|
||||||
if [ -n "$USER" ]; then
|
|
||||||
DST_HOST="$USER@$DST_HOST"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$CEPH_USER" ]; then
|
|
||||||
RBD="$RBD --id '$(esc_sq "${CEPH_USER}")'"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$CEPH_KEY" ]; then
|
|
||||||
RBD="$RBD --keyfile '$(esc_sq "${CEPH_KEY}")'"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$CEPH_CONF" ]; then
|
|
||||||
RBD="$RBD --conf '$(esc_sq "${CEPH_CONF}")'"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "ssh '$(esc_sq "$DST_HOST")' \"$RBD export '$(esc_sq "$SOURCE")' -\""
|
|
||||||
}
|
|
||||||
|
|
||||||
function get_vitastor_cmd
|
|
||||||
{
|
|
||||||
local i j URL_ELEMENTS
|
|
||||||
|
|
||||||
FROM="$1"
|
|
||||||
|
|
||||||
URL_RB="$DRIVER_PATH/url.rb"
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
URL_ELEMENTS[i++]="$element"
|
|
||||||
done < <($URL_RB "$FROM" \
|
|
||||||
USER \
|
|
||||||
HOST \
|
|
||||||
SOURCE \
|
|
||||||
PARAM_DS \
|
|
||||||
PARAM_VITASTOR_CONF)
|
|
||||||
|
|
||||||
USER="${URL_ELEMENTS[j++]}"
|
|
||||||
DST_HOST="${URL_ELEMENTS[j++]}"
|
|
||||||
SOURCE="${URL_ELEMENTS[j++]}"
|
|
||||||
DS="${URL_ELEMENTS[j++]}"
|
|
||||||
VITASTOR_CONF="${URL_ELEMENTS[j++]}"
|
|
||||||
|
|
||||||
# Remove leading '/'
|
|
||||||
SOURCE="${SOURCE#/}"
|
|
||||||
|
|
||||||
if [ -n "$USER" ]; then
|
|
||||||
DST_HOST="$USER@$DST_HOST"
|
|
||||||
fi
|
|
||||||
|
|
||||||
local CLI
|
|
||||||
CLI="vitastor-cli"
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path '$(esc_sq "${VITASTOR_CONF}")'"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "ssh '$(esc_sq "$DST_HOST")' \"$CLI dd iimg='$(esc_sq "$SOURCE")'\""
|
|
||||||
}
|
|
||||||
|
|
||||||
# Compare 2 version strings using sort -V
|
|
||||||
# Usage:
|
|
||||||
# verlte "3.2.9" "3.4.0"
|
|
||||||
function verlte() {
|
|
||||||
[ "$1" = "`echo -e "$1\n$2" | sort -V | head -n1`" ]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Returns curl retry options based on its version
|
|
||||||
function curl_retry_args {
|
|
||||||
[ "$NO_RETRY" = "yes" ] && return
|
|
||||||
|
|
||||||
RETRY_ARGS="--retry 3 --retry-delay 3"
|
|
||||||
|
|
||||||
CURL_VER=`curl --version | grep -o 'curl [0-9\.]*' | awk '{print $2}'`
|
|
||||||
|
|
||||||
# To retry also on conn-reset-by-peer fresh curl is needed
|
|
||||||
if verlte "7.71.0" "$CURL_VER" && [ -z ${MAX_SIZE} ] ; then
|
|
||||||
RETRY_ARGS+=" --retry-all-errors"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo $RETRY_ARGS
|
|
||||||
}
|
|
||||||
|
|
||||||
TEMP=`getopt -o m:s:l:c:no -l md5:,sha1:,limit:,max-size:,nodecomp,noretry -- "$@"`
|
|
||||||
|
|
||||||
if [ $? != 0 ] ; then
|
|
||||||
echo "Arguments error" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
eval set -- "$TEMP"
|
|
||||||
|
|
||||||
while true; do
|
|
||||||
case "$1" in
|
|
||||||
-m|--md5)
|
|
||||||
HASH_TYPE=md5
|
|
||||||
HASH=$2
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-s|--sha1)
|
|
||||||
HASH_TYPE=sha1
|
|
||||||
HASH=$2
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-n|--nodecomp)
|
|
||||||
export NO_DECOMPRESS="yes"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
-l|--limit)
|
|
||||||
export LIMIT_RATE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-c|--max-size)
|
|
||||||
export MAX_SIZE="$2"
|
|
||||||
shift 2
|
|
||||||
;;
|
|
||||||
-o|--noretry)
|
|
||||||
export NO_RETRY="yes"
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
--)
|
|
||||||
shift
|
|
||||||
break
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
shift
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
FROM="$1"
|
|
||||||
TO="$2"
|
|
||||||
|
|
||||||
if [ -n "${HASH_TYPE}" -a -n "${MAX_SIZE}" ]; then
|
|
||||||
echo "Hash check not supported for partial downloads" >&2
|
|
||||||
exit -1
|
|
||||||
else
|
|
||||||
# File used by the hasher function to store the resulting hash
|
|
||||||
export HASH_FILE="/tmp/downloader.hash.$$"
|
|
||||||
fi
|
|
||||||
|
|
||||||
GLOBAL_CURL_ARGS="--fail -sS -k -L $(curl_retry_args)"
|
|
||||||
|
|
||||||
case "$FROM" in
|
|
||||||
http://*|https://*)
|
|
||||||
# -k so it does not check the certificate
|
|
||||||
# -L to follow redirects
|
|
||||||
# -sS to hide output except on failure
|
|
||||||
# --limit_rate to limit the bw
|
|
||||||
curl_args="$GLOBAL_CURL_ARGS '$(esc_sq "${FROM}")'"
|
|
||||||
|
|
||||||
if [ -n "$LIMIT_RATE" ]; then
|
|
||||||
curl_args="--limit-rate $LIMIT_RATE $curl_args"
|
|
||||||
fi
|
|
||||||
|
|
||||||
command="curl $curl_args"
|
|
||||||
;;
|
|
||||||
ssh://*)
|
|
||||||
# pseudo-url for ssh transfers ssh://user@host:path
|
|
||||||
# -l to limit the bw
|
|
||||||
ssh_src=${FROM#ssh://}
|
|
||||||
ssh_arg=(${ssh_src/:/ })
|
|
||||||
|
|
||||||
rmt_cmd="\"cat '$(esc_sq "${ssh_arg[1]}")'\""
|
|
||||||
|
|
||||||
command="ssh ${ssh_arg[0]} $rmt_cmd"
|
|
||||||
;;
|
|
||||||
s3://*)
|
|
||||||
# Read s3 environment
|
|
||||||
s3_env
|
|
||||||
|
|
||||||
if [ -z "$S3_ACCESS_KEY_ID" -o -z "$S3_SECRET_ACCESS_KEY" ]; then
|
|
||||||
echo "S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY are required" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
curl_args=""
|
|
||||||
|
|
||||||
if [[ "$S3_AWS" =~ (no|NO) ]]; then
|
|
||||||
curl_args="$(s3_curl_args "$FROM")"
|
|
||||||
else
|
|
||||||
curl_args="$(s3_curl_args_aws "$FROM")"
|
|
||||||
fi
|
|
||||||
|
|
||||||
command="curl $GLOBAL_CURL_ARGS $curl_args"
|
|
||||||
;;
|
|
||||||
rbd://*)
|
|
||||||
command="$(get_rbd_cmd "$FROM")"
|
|
||||||
;;
|
|
||||||
vitastor://*)
|
|
||||||
command="$(get_vitastor_cmd "$FROM")"
|
|
||||||
;;
|
|
||||||
vcenter://*)
|
|
||||||
command="$VAR_LOCATION/remotes/datastore/vcenter_downloader.rb '$(esc_sq "$FROM")'"
|
|
||||||
;;
|
|
||||||
lxd://*)
|
|
||||||
file_type="application/octet-stream"
|
|
||||||
command="$VAR_LOCATION/remotes/datastore/lxd_downloader.sh \"$FROM\""
|
|
||||||
;;
|
|
||||||
restic://*)
|
|
||||||
eval `$VAR_LOCATION/remotes/datastore/restic_downloader.rb "$FROM" | grep -e '^command=' -e '^clean_command='`
|
|
||||||
;;
|
|
||||||
rsync://*)
|
|
||||||
eval `$VAR_LOCATION/remotes/datastore/rsync_downloader.rb "$FROM" | grep -e '^command=' -e '^clean_command='`
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
if [ ! -r $FROM ]; then
|
|
||||||
echo "Cannot read from $FROM" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
command="cat '$(esc_sq "$FROM")'"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
[ -z "$file_type" ] && file_type=$(get_type "$command")
|
|
||||||
decompressor=$(get_decompressor "$file_type")
|
|
||||||
|
|
||||||
if [ -z "${MAX_SIZE}" ]; then
|
|
||||||
eval "$command" | \
|
|
||||||
tee >( hasher $HASH_TYPE) | \
|
|
||||||
decompress "$decompressor" "$TO"
|
|
||||||
|
|
||||||
if [ "$?" != "0" -o "$PIPESTATUS" != "0" ]; then
|
|
||||||
echo "Error copying" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
else
|
|
||||||
# Order of the 'head' command is here on purpose:
|
|
||||||
# 1. We want to download more bytes than needed to get a requested
|
|
||||||
# number of bytes on the output. Decompressor may need more
|
|
||||||
# data to decompress the stream.
|
|
||||||
# 2. Decompressor command is also misused to detect SIGPIPE error.
|
|
||||||
eval "$command" | \
|
|
||||||
decompress "$decompressor" "$TO" 2>/dev/null | \
|
|
||||||
head -c "${MAX_SIZE}"
|
|
||||||
|
|
||||||
# Following table shows exit codes of each command
|
|
||||||
# in the pipe for various scenarios:
|
|
||||||
#
|
|
||||||
# ----------------------------------------------------
|
|
||||||
# | $COMMAND | TYPE | PIPESTATUS | BEHAVIOUR
|
|
||||||
# ----------------------------------------------------
|
|
||||||
# | cat | partial | 141 141 0 | OK
|
|
||||||
# | cat | full | 0 0 0 | OK
|
|
||||||
# | cat | error | 1 0 0 | fail
|
|
||||||
# | curl | partial | 23 141 0 | OK
|
|
||||||
# | curl | full | 0 0 0 | OK
|
|
||||||
# | curl | error | 22 0 0 | fail
|
|
||||||
# | ssh | partial | 255 141 0 | OK
|
|
||||||
# | ssh | full | 0 0 0 | OK
|
|
||||||
# | ssh | error ssh | 255 0 0 | fail
|
|
||||||
# | ssh | error ssh cat | 1 0 0 | fail
|
|
||||||
if [ \( "${PIPESTATUS[0]}" != '0' -a "${PIPESTATUS[1]}" = '0' \) \
|
|
||||||
-o \( "${PIPESTATUS[1]}" != '0' -a "${PIPESTATUS[1]}" != '141' \) \
|
|
||||||
-o \( "${PIPESTATUS[2]}" != "0" \) ];
|
|
||||||
then
|
|
||||||
echo "Error copying" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$HASH_TYPE" ]; then
|
|
||||||
HASH_RESULT=$( cat $HASH_FILE)
|
|
||||||
rm $HASH_FILE
|
|
||||||
if [ "$HASH_RESULT" != "$HASH" ]; then
|
|
||||||
echo "Hash does not match" >&2
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Unarchive only if the destination is filesystem
|
|
||||||
if [ "$TO" != "-" ]; then
|
|
||||||
unarchive "$TO"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Perform any clean operation
|
|
||||||
if [ -n "${clean_command}" ]; then
|
|
||||||
eval "$clean_command"
|
|
||||||
fi
|
|
|
@ -1,60 +0,0 @@
|
||||||
diff --git /var/lib/one/remotes/datastore/downloader.sh /var/lib/one/remotes/datastore/downloader.sh
|
|
||||||
index 9b75d8ee4b..09d2a5d41d 100755
|
|
||||||
--- /var/lib/one/remotes/datastore/downloader.sh
|
|
||||||
+++ /var/lib/one/remotes/datastore/downloader.sh
|
|
||||||
@@ -295,6 +295,45 @@ function get_rbd_cmd
|
|
||||||
echo "ssh '$(esc_sq "$DST_HOST")' \"$RBD export '$(esc_sq "$SOURCE")' -\""
|
|
||||||
}
|
|
||||||
|
|
||||||
+function get_vitastor_cmd
|
|
||||||
+{
|
|
||||||
+ local i j URL_ELEMENTS
|
|
||||||
+
|
|
||||||
+ FROM="$1"
|
|
||||||
+
|
|
||||||
+ URL_RB="$DRIVER_PATH/url.rb"
|
|
||||||
+
|
|
||||||
+ while IFS= read -r -d '' element; do
|
|
||||||
+ URL_ELEMENTS[i++]="$element"
|
|
||||||
+ done < <($URL_RB "$FROM" \
|
|
||||||
+ USER \
|
|
||||||
+ HOST \
|
|
||||||
+ SOURCE \
|
|
||||||
+ PARAM_DS \
|
|
||||||
+ PARAM_VITASTOR_CONF)
|
|
||||||
+
|
|
||||||
+ USER="${URL_ELEMENTS[j++]}"
|
|
||||||
+ DST_HOST="${URL_ELEMENTS[j++]}"
|
|
||||||
+ SOURCE="${URL_ELEMENTS[j++]}"
|
|
||||||
+ DS="${URL_ELEMENTS[j++]}"
|
|
||||||
+ VITASTOR_CONF="${URL_ELEMENTS[j++]}"
|
|
||||||
+
|
|
||||||
+ # Remove leading '/'
|
|
||||||
+ SOURCE="${SOURCE#/}"
|
|
||||||
+
|
|
||||||
+ if [ -n "$USER" ]; then
|
|
||||||
+ DST_HOST="$USER@$DST_HOST"
|
|
||||||
+ fi
|
|
||||||
+
|
|
||||||
+ local CLI
|
|
||||||
+ CLI="vitastor-cli"
|
|
||||||
+ if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
+ CLI="$CLI --config_path '$(esc_sq "${VITASTOR_CONF}")'"
|
|
||||||
+ fi
|
|
||||||
+
|
|
||||||
+ echo "ssh '$(esc_sq "$DST_HOST")' \"$CLI dd iimg='$(esc_sq "$SOURCE")'\""
|
|
||||||
+}
|
|
||||||
+
|
|
||||||
# Compare 2 version strings using sort -V
|
|
||||||
# Usage:
|
|
||||||
# verlte "3.2.9" "3.4.0"
|
|
||||||
@@ -424,6 +463,9 @@ s3://*)
|
|
||||||
rbd://*)
|
|
||||||
command="$(get_rbd_cmd "$FROM")"
|
|
||||||
;;
|
|
||||||
+vitastor://*)
|
|
||||||
+ command="$(get_vitastor_cmd "$FROM")"
|
|
||||||
+ ;;
|
|
||||||
vcenter://*)
|
|
||||||
command="$VAR_LOCATION/remotes/datastore/vcenter_downloader.rb '$(esc_sq "$FROM")'"
|
|
||||||
;;
|
|
|
@ -1,114 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Vitastor OpenNebula driver
|
|
||||||
# Copyright (c) Vitaliy Filippov, 2024+
|
|
||||||
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
# This script is used to export an image to qcow2 file
|
|
||||||
|
|
||||||
# ------------ Set up the environment to source common tools ------------
|
|
||||||
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
source ${DRIVER_PATH}/../libfs.sh
|
|
||||||
|
|
||||||
# -------- Get rm and datastore arguments from OpenNebula core ------------
|
|
||||||
|
|
||||||
DRV_ACTION=`cat -`
|
|
||||||
ID=$1
|
|
||||||
|
|
||||||
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/SOURCE \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/MD5 \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/SHA1 \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/TEMPLATE/FORMAT \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
|
|
||||||
|
|
||||||
unset i
|
|
||||||
|
|
||||||
SRC="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SIZE="${XPATH_ELEMENTS[i++]}"
|
|
||||||
MD5="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SHA1="${XPATH_ELEMENTS[i++]}"
|
|
||||||
FORMAT="${XPATH_ELEMENTS[i++]:-raw}"
|
|
||||||
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
|
|
||||||
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
|
|
||||||
|
|
||||||
DST_HOST=`get_destination_host $ID`
|
|
||||||
|
|
||||||
if [ -z "$DST_HOST" ]; then
|
|
||||||
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
IMPORT_SOURCE="vitastor://$DST_HOST/$SRC"
|
|
||||||
IS_JOIN="?"
|
|
||||||
|
|
||||||
CLI=vitastor-cli
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path $VITASTOR_CONF"
|
|
||||||
IMPORT_SOURCE="${IMPORT_SOURCE}${IS_JOIN}VITASTOR_CONF=${VITASTOR_CONF}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# FIXME: this is inefficient - it pipes the image twice...
|
|
||||||
|
|
||||||
INFO_SCRIPT=$(cat <<EOF
|
|
||||||
if [ -z "$MD5" ]; then
|
|
||||||
CHECKSUM=\$(
|
|
||||||
$CLI dd iimg=${SRC} | ${MD5SUM} | cut -f1 -d' '
|
|
||||||
ps=\$PIPESTATUS
|
|
||||||
|
|
||||||
if [ "\$ps" != "0" ]; then
|
|
||||||
exit \$ps
|
|
||||||
fi
|
|
||||||
)
|
|
||||||
|
|
||||||
status=\$?
|
|
||||||
|
|
||||||
[ "\$status" != "0" ] && exit \$status
|
|
||||||
else
|
|
||||||
CHECKSUM="$MD5"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "\$CHECKSUM" ]; then
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
cat <<EOT
|
|
||||||
<MD5><![CDATA[\$CHECKSUM]]></MD5>
|
|
||||||
<SIZE><![CDATA[$SIZE]]></SIZE>
|
|
||||||
<FORMAT><![CDATA[${FORMAT}]]></FORMAT>
|
|
||||||
EOT
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
INFO=$(ssh_monitor_and_log "$DST_HOST" "$INFO_SCRIPT" "Image info script" 2>&1)
|
|
||||||
INFO_STATUS=$?
|
|
||||||
|
|
||||||
if [ "$INFO_STATUS" != "0" ]; then
|
|
||||||
echo "$INFO"
|
|
||||||
exit $INFO_STATUS
|
|
||||||
fi
|
|
||||||
|
|
||||||
cat <<EOF
|
|
||||||
<IMPORT_INFO>
|
|
||||||
<IMPORT_SOURCE><![CDATA[$IMPORT_SOURCE]]></IMPORT_SOURCE>
|
|
||||||
$INFO
|
|
||||||
<DISPOSE>NO</DISPOSE>
|
|
||||||
</IMPORT_INFO>"
|
|
||||||
EOF
|
|
|
@ -1,124 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Vitastor OpenNebula driver
|
|
||||||
# Copyright (c) Vitaliy Filippov, 2024+
|
|
||||||
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
# This script is used to create a VM image (SRC) of size (SIZE) and formatted as (FS)
|
|
||||||
|
|
||||||
# -------- Set up the environment to source common tools & conf ------------
|
|
||||||
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
source ${DRIVER_PATH}/../libfs.sh
|
|
||||||
source ${DRIVER_PATH}/../../etc/datastore/datastore.conf
|
|
||||||
|
|
||||||
# -------- Get mkfs and datastore arguments from OpenNebula core ------------
|
|
||||||
|
|
||||||
DRV_ACTION=`cat -`
|
|
||||||
ID=$1
|
|
||||||
|
|
||||||
XPATH="${DRIVER_PATH}/../xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/BASE_PATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/RESTRICTED_DIRS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/SAFE_DIRS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/IMAGE_PREFIX \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/FORMAT \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/SIZE \
|
|
||||||
/DS_DRIVER_ACTION_DATA/IMAGE/FS \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
|
|
||||||
|
|
||||||
unset i
|
|
||||||
|
|
||||||
BASE_PATH="${XPATH_ELEMENTS[i++]}"
|
|
||||||
RESTRICTED_DIRS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SAFE_DIRS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
BRIDGE_LIST="${XPATH_ELEMENTS[i++]}"
|
|
||||||
POOL_NAME="${XPATH_ELEMENTS[i++]}"
|
|
||||||
IMAGE_PREFIX="${XPATH_ELEMENTS[i++]:-one}"
|
|
||||||
FORMAT="${XPATH_ELEMENTS[i++]}"
|
|
||||||
SIZE="${XPATH_ELEMENTS[i++]}"
|
|
||||||
FS="${XPATH_ELEMENTS[i++]}"
|
|
||||||
VITASTOR_CONF="${XPATH_ELEMENTS[i++]}"
|
|
||||||
|
|
||||||
DST_HOST=`get_destination_host $ID`
|
|
||||||
|
|
||||||
if [ -z "$DST_HOST" ]; then
|
|
||||||
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLI=
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path ${VITASTOR_CONF}"
|
|
||||||
fi
|
|
||||||
if [ -n "$POOL_NAME" ]; then
|
|
||||||
CLI="$CLI --pool ${POOL_NAME}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
set_up_datastore "$BASE_PATH" "$RESTRICTED_DIRS" "$SAFE_DIRS"
|
|
||||||
|
|
||||||
IMAGE_NAME="${IMAGE_PREFIX}-${ID}"
|
|
||||||
|
|
||||||
# ------------ Image to save_as disk, no need to create a new image ------------
|
|
||||||
|
|
||||||
if [ "$FORMAT" = "save_as" ]; then
|
|
||||||
echo "$IMAGE_NAME"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ------------ Create the image in the repository ------------
|
|
||||||
|
|
||||||
# FIXME: Duplicate code with tm/vitastor/mkimage
|
|
||||||
|
|
||||||
MKIMAGE_CMD=$(cat <<EOF
|
|
||||||
set -e -o pipefail
|
|
||||||
export PATH=/usr/sbin:/sbin:\$PATH
|
|
||||||
vitastor-cli $CLI create --pool "${POOL_NAME}" "$IMAGE_NAME" --size "${SIZE}M"
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
if [ -n "$FS" -o "$FORMAT" = "swap" ]; then
|
|
||||||
MKFS_CMD=`mkfs_command '$NBD' raw "$SIZE" "$SUPPORTED_FS" "$FS" "$FS_OPTS" | grep -v $QEMU_IMG`
|
|
||||||
fi
|
|
||||||
|
|
||||||
MKIMAGE_CMD=$(cat <<EOF
|
|
||||||
set -e -o pipefail
|
|
||||||
export PATH=/usr/sbin:/sbin:\$PATH
|
|
||||||
vitastor-cli $CLI create --pool "${POOL_NAME}" "$IMAGE_NAME" --size "${SIZE}M"
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
if [ ! -z $FS ]; then
|
|
||||||
set -e -o pipefail
|
|
||||||
|
|
||||||
IMAGE_HASH=`generate_image_hash`
|
|
||||||
FS_OPTS=$(eval $(echo "echo \$FS_OPTS_$FS"))
|
|
||||||
|
|
||||||
MKFS_CMD=`mkfs_command '$NBD' raw "$SIZE" "$SUPPORTED_FS" "$FS" "$FS_OPTS" | grep -v $QEMU_IMG`
|
|
||||||
MKIMAGE_CMD=$(cat <<EOF
|
|
||||||
$MKIMAGE_CMD
|
|
||||||
NBD=\$(sudo vitastor-nbd $CLI map --image "$IMAGE_NAME")
|
|
||||||
trap "sudo vitastor-nbd $CLI unmap \$NBD" EXIT
|
|
||||||
$MKFS_CMD
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
fi
|
|
||||||
|
|
||||||
ssh_exec_and_log "$DST_HOST" "$MKIMAGE_CMD" "Error registering $IMAGE_NAME in $DST_HOST"
|
|
||||||
|
|
||||||
echo "$IMAGE_NAME"
|
|
|
@ -1,64 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
# Vitastor OpenNebula driver
|
|
||||||
# Copyright (c) Vitaliy Filippov, 2024+
|
|
||||||
# License: Apache-2.0 http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
|
|
||||||
# This script is used to monitor the free and used space of a datastore
|
|
||||||
|
|
||||||
# -------- Set up the environment to source common tools & conf ------------
|
|
||||||
|
|
||||||
if [ -z "${ONE_LOCATION}" ]; then
|
|
||||||
LIB_LOCATION=/usr/lib/one
|
|
||||||
else
|
|
||||||
LIB_LOCATION=$ONE_LOCATION/lib
|
|
||||||
fi
|
|
||||||
|
|
||||||
. $LIB_LOCATION/sh/scripts_common.sh
|
|
||||||
|
|
||||||
DRIVER_PATH=$(dirname $0)
|
|
||||||
source ${DRIVER_PATH}/../../datastore/libfs.sh
|
|
||||||
|
|
||||||
# -------- Get datastore arguments from OpenNebula core ------------
|
|
||||||
|
|
||||||
DRV_ACTION=`cat -`
|
|
||||||
ID=$1
|
|
||||||
|
|
||||||
XPATH="${DRIVER_PATH}/../../datastore/xpath.rb -b $DRV_ACTION"
|
|
||||||
|
|
||||||
unset i j XPATH_ELEMENTS
|
|
||||||
|
|
||||||
while IFS= read -r -d '' element; do
|
|
||||||
XPATH_ELEMENTS[i++]="$element"
|
|
||||||
done < <($XPATH \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/BRIDGE_LIST \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/POOL_NAME \
|
|
||||||
/DS_DRIVER_ACTION_DATA/DATASTORE/TEMPLATE/VITASTOR_CONF)
|
|
||||||
|
|
||||||
BRIDGE_LIST="${XPATH_ELEMENTS[j++]}"
|
|
||||||
POOL_NAME="${XPATH_ELEMENTS[j++]}"
|
|
||||||
VITASTOR_CONF="${XPATH_ELEMENTS[j++]}"
|
|
||||||
|
|
||||||
HOST=`get_destination_host`
|
|
||||||
|
|
||||||
if [ -z "$HOST" ]; then
|
|
||||||
error_message "Datastore template missing 'BRIDGE_LIST' attribute."
|
|
||||||
exit -1
|
|
||||||
fi
|
|
||||||
|
|
||||||
CLI=vitastor-cli
|
|
||||||
if [ -n "$VITASTOR_CONF" ]; then
|
|
||||||
CLI="$CLI --config_path ${VITASTOR_CONF}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# ------------ Compute datastore usage -------------
|
|
||||||
|
|
||||||
MONITOR_SCRIPT=$(cat <<EOF
|
|
||||||
vitastor-cli df --json | jq -r '.[] | select(.name == "${POOL_NAME}") |
|
|
||||||
"TOTAL_MB="+(.total_raw/.raw_to_usable/1024/1024 | tostring)+
|
|
||||||
"\nUSED_MB="+(.used_raw/.raw_to_usable/1024/1024 | tostring)+
|
|
||||||
"\nFREE_MB="+(.max_available/1024/1024 | tostring)'
|
|
||||||
EOF
|
|
||||||
)
|
|
||||||
|
|
||||||
ssh_monitor_and_log $HOST "$MONITOR_SCRIPT 2>&1" "Error monitoring ${POOL_NAME} in $HOST"
|
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue