Compare commits
103 Commits
f416abb4bb
...
416924f6a8
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | 416924f6a8 | |
Vitaliy Filippov | c57ea9be61 | |
Vitaliy Filippov | 7eef87aaa1 | |
Vitaliy Filippov | ca01931e72 | |
Vitaliy Filippov | 57eb811bb1 | |
Vitaliy Filippov | 5d3aaf016b | |
Vitaliy Filippov | 0b097ca3f2 | |
Vitaliy Filippov | 989675a780 | |
Vitaliy Filippov | f8c403ec9e | |
Vitaliy Filippov | bfbb85e653 | |
Vitaliy Filippov | 9ad6822353 | |
Vitaliy Filippov | 2043b4e374 | |
Vitaliy Filippov | de840e6fe3 | |
Vitaliy Filippov | b5e04bf809 | |
Vitaliy Filippov | 8807a1623b | |
Vitaliy Filippov | f12855c31b | |
Vitaliy Filippov | e75dcc9a71 | |
Vitaliy Filippov | 88516ab4bd | |
Vitaliy Filippov | 6221126b4f | |
Vitaliy Filippov | 6783d4a13c | |
Vitaliy Filippov | dcbe1afac3 | |
Vitaliy Filippov | 0bde28c24a | |
Vitaliy Filippov | bb8ca6184e | |
Vitaliy Filippov | 87310ef7bb | |
Vitaliy Filippov | 4f4b2dab80 | |
Vitaliy Filippov | f70da82317 | |
Vitaliy Filippov | e42148f347 | |
Vitaliy Filippov | c289584469 | |
Vitaliy Filippov | 018e89f867 | |
Vitaliy Filippov | 603dc68f11 | |
Vitaliy Filippov | 7b12342933 | |
Vitaliy Filippov | 44bf0f16ee | |
Vitaliy Filippov | 8840c84572 | |
Vitaliy Filippov | 5b747c12ec | |
Vitaliy Filippov | 05f5f46162 | |
Vitaliy Filippov | b5604191c8 | |
Vitaliy Filippov | e871de27de | |
Vitaliy Filippov | f600ce98e2 | |
Vitaliy Filippov | 57605a5c13 | |
Vitaliy Filippov | 29bd4561bb | |
Vitaliy Filippov | 7142460ec8 | |
Vitaliy Filippov | d03f19ebe5 | |
Vitaliy Filippov | 88f9d18be3 | |
Vitaliy Filippov | 6213fbd8c6 | |
Vitaliy Filippov | 3aee37eadd | |
Vitaliy Filippov | ecfc753e93 | |
Vitaliy Filippov | a574f9ad71 | |
Vitaliy Filippov | 7c235c9103 | |
Vitaliy Filippov | e5bb986164 | |
Vitaliy Filippov | 181795d748 | |
Vitaliy Filippov | 8cdc38805b | |
Vitaliy Filippov | 0cd455d17f | |
Vitaliy Filippov | 32ba653ba6 | |
Vitaliy Filippov | 231d4b15fc | |
Vitaliy Filippov | 9dc4d5fd7b | |
Vitaliy Filippov | e58538fa47 | |
Vitaliy Filippov | 11ac9e7024 | |
Vitaliy Filippov | 511bc3df1c | |
Vitaliy Filippov | a64f0d1f73 | |
Vitaliy Filippov | ec5f7c6b87 | |
Vitaliy Filippov | 3ebed9a749 | |
Vitaliy Filippov | eab67a6e8f | |
Vitaliy Filippov | 20993d9b7a | |
Vitaliy Filippov | 5cf9b343c0 | |
Vitaliy Filippov | 79ae0aadcd | |
Vitaliy Filippov | 605afc3583 | |
Vitaliy Filippov | c0681d8242 | |
Vitaliy Filippov | 763e77b4f4 | |
Vitaliy Filippov | 19426aa4c5 | |
Vitaliy Filippov | 08f586bcec | |
Vitaliy Filippov | f1cd87473a | |
Vitaliy Filippov | 1bd8d2da56 | |
Vitaliy Filippov | a7396d2baf | |
Vitaliy Filippov | e98a38810d | |
Vitaliy Filippov | 28c4324c36 | |
Vitaliy Filippov | 31ec3fa8f5 | |
Vitaliy Filippov | e4fa26f60a | |
Vitaliy Filippov | 59ae27f9e5 | |
Vitaliy Filippov | 2c6a301d9b | |
Vitaliy Filippov | 01558349f8 | |
Vitaliy Filippov | 36f4717d0d | |
Vitaliy Filippov | babaf2a0ce | |
Vitaliy Filippov | 5773f1a375 | |
Vitaliy Filippov | 57222a9f79 | |
Vitaliy Filippov | 61ef000c6e | |
Vitaliy Filippov | 7d5e1cc393 | |
Vitaliy Filippov | 5e7f27a02d | |
Vitaliy Filippov | fd1d8a8520 | |
Vitaliy Filippov | c364e14c40 | |
Vitaliy Filippov | 3ebbfa0428 | |
Vitaliy Filippov | aa79d1db1c | |
Vitaliy Filippov | a1fecb7eff | |
Vitaliy Filippov | ff74b19423 | |
Vitaliy Filippov | 4cf6dceed7 | |
Vitaliy Filippov | 38b8963330 | |
Vitaliy Filippov | 77167e2920 | |
Vitaliy Filippov | 5af23672d0 | |
Vitaliy Filippov | 6bf1f539a6 | |
Vitaliy Filippov | 4eab26f968 | |
Vitaliy Filippov | 86243b7101 | |
idelson | dc92851322 | |
Zibort Cloud | 02d1f16bbd | |
Vitaliy Filippov | fc413038d1 |
|
@ -22,7 +22,7 @@ RUN apt-get update
|
|||
RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
|
||||
liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
|
||||
RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
|
||||
RUN apt-get -y install jq lp-solve sudo
|
||||
RUN apt-get -y install jq lp-solve sudo nfs-common
|
||||
RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
|
||||
|
||||
RUN set -ex; \
|
||||
|
|
|
@ -856,3 +856,21 @@ jobs:
|
|||
echo ""
|
||||
done
|
||||
|
||||
test_nfs:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_nfs.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
|
|
|
@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
|
|||
|
||||
project(vitastor)
|
||||
|
||||
set(VERSION "1.4.7")
|
||||
set(VERSION "1.5.0")
|
||||
|
||||
add_subdirectory(src)
|
||||
|
|
|
@ -6,8 +6,8 @@
|
|||
|
||||
Вернём былую скорость кластерному блочному хранилищу!
|
||||
|
||||
Vitastor - распределённая блочная SDS (программная СХД), прямой аналог Ceph RBD и
|
||||
внутренних СХД популярных облачных провайдеров. Однако, в отличие от них, Vitastor
|
||||
Vitastor - распределённая блочная и файловая SDS (программная СХД), прямой аналог Ceph RBD и CephFS,
|
||||
а также внутренних СХД популярных облачных провайдеров. Однако, в отличие от них, Vitastor
|
||||
быстрый и при этом простой. Только пока маленький :-).
|
||||
|
||||
Vitastor архитектурно похож на Ceph, что означает атомарность и строгую консистентность,
|
||||
|
@ -63,11 +63,13 @@ Vitastor поддерживает QEMU-драйвер, протоколы NBD и
|
|||
- [fio](docs/usage/fio.ru.md) для тестов производительности
|
||||
- [NBD](docs/usage/nbd.ru.md) для монтирования ядром
|
||||
- [QEMU и qemu-img](docs/usage/qemu.ru.md)
|
||||
- [NFS](docs/usage/nfs.ru.md)-прокси для VMWare и подобных
|
||||
- [NFS](docs/usage/nfs.ru.md) кластерная файловая система и псевдо-ФС прокси
|
||||
- [Администрирование](docs/usage/admin.ru.md)
|
||||
- Производительность
|
||||
- [Понимание сути производительности](docs/performance/understanding.ru.md)
|
||||
- [Теоретический максимум](docs/performance/theoretical.ru.md)
|
||||
- [Пример сравнения с Ceph](docs/performance/comparison1.ru.md)
|
||||
- [Более новый тест Vitastor 1.3.1](docs/performance/bench2.ru.md)
|
||||
|
||||
## Автор и лицензия
|
||||
|
||||
|
|
10
README.md
10
README.md
|
@ -6,9 +6,9 @@
|
|||
|
||||
Make Clustered Block Storage Fast Again.
|
||||
|
||||
Vitastor is a distributed block SDS, direct replacement of Ceph RBD and internal SDS's
|
||||
of public clouds. However, in contrast to them, Vitastor is fast and simple at the same time.
|
||||
The only thing is it's slightly young :-).
|
||||
Vitastor is a distributed block and file SDS, direct replacement of Ceph RBD and CephFS,
|
||||
and also internal SDS's of public clouds. However, in contrast to them, Vitastor is fast
|
||||
and simple at the same time. The only thing is it's slightly young :-).
|
||||
|
||||
Vitastor is architecturally similar to Ceph which means strong consistency,
|
||||
primary-replication, symmetric clustering and automatic data distribution over any
|
||||
|
@ -63,11 +63,13 @@ Read more details below in the documentation.
|
|||
- [fio](docs/usage/fio.en.md) for benchmarks
|
||||
- [NBD](docs/usage/nbd.en.md) for kernel mounts
|
||||
- [QEMU and qemu-img](docs/usage/qemu.en.md)
|
||||
- [NFS](docs/usage/nfs.en.md) emulator for VMWare and similar
|
||||
- [NFS](docs/usage/nfs.en.md) clustered file system and pseudo-FS proxy
|
||||
- [Administration](docs/usage/admin.en.md)
|
||||
- Performance
|
||||
- [Understanding storage performance](docs/performance/understanding.en.md)
|
||||
- [Theoretical performance](docs/performance/theoretical.en.md)
|
||||
- [Example comparison with Ceph](docs/performance/comparison1.en.md)
|
||||
- [Newer benchmark of Vitastor 1.3.1](docs/performance/bench2.en.md)
|
||||
|
||||
## Author and License
|
||||
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
VERSION ?= v1.4.7
|
||||
VERSION ?= v1.5.0
|
||||
|
||||
all: build push
|
||||
|
||||
|
|
|
@ -49,7 +49,7 @@ spec:
|
|||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
allowPrivilegeEscalation: true
|
||||
image: vitalif/vitastor-csi:v1.4.7
|
||||
image: vitalif/vitastor-csi:v1.5.0
|
||||
args:
|
||||
- "--node=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
|
|
|
@ -121,7 +121,7 @@ spec:
|
|||
privileged: true
|
||||
capabilities:
|
||||
add: ["SYS_ADMIN"]
|
||||
image: vitalif/vitastor-csi:v1.4.7
|
||||
image: vitalif/vitastor-csi:v1.5.0
|
||||
args:
|
||||
- "--node=$(NODE_ID)"
|
||||
- "--endpoint=$(CSI_ENDPOINT)"
|
||||
|
|
|
@ -5,7 +5,7 @@ package vitastor
|
|||
|
||||
const (
|
||||
vitastorCSIDriverName = "csi.vitastor.io"
|
||||
vitastorCSIDriverVersion = "1.4.7"
|
||||
vitastorCSIDriverVersion = "1.5.0"
|
||||
)
|
||||
|
||||
// Config struct fills the parameters of request or user input
|
||||
|
|
|
@ -3,5 +3,5 @@
|
|||
cat < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build --build-arg REL=bookworm -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
sudo podman build --build-arg DISTRO=debian --build-arg REL=bookworm -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
||||
|
|
|
@ -3,5 +3,5 @@
|
|||
cat < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
sudo podman build --build-arg DISTRO=debian --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
||||
|
|
|
@ -3,5 +3,5 @@
|
|||
cat < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build --build-arg REL=buster -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
sudo podman build --build-arg DISTRO=debian --build-arg REL=buster -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
||||
|
|
|
@ -0,0 +1,7 @@
|
|||
#!/bin/bash
|
||||
|
||||
cat < vitastor.Dockerfile > ../Dockerfile
|
||||
cd ..
|
||||
mkdir -p packages
|
||||
sudo podman build --build-arg DISTRO=ubuntu --build-arg REL=jammy -v `pwd`/packages:/root/packages -f Dockerfile .
|
||||
rm Dockerfile
|
|
@ -1,4 +1,4 @@
|
|||
vitastor (1.4.7-1) unstable; urgency=medium
|
||||
vitastor (1.5.0-1) unstable; urgency=medium
|
||||
|
||||
* Bugfixes
|
||||
|
||||
|
|
|
@ -1,13 +1,14 @@
|
|||
# Build patched libvirt for Debian Buster or Bullseye/Sid inside a container
|
||||
# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/libvirt.Dockerfile .
|
||||
# cd ..; podman build --build-arg DISTRO=debian --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/libvirt.Dockerfile .
|
||||
|
||||
ARG DISTRO=
|
||||
ARG REL=
|
||||
FROM debian:$REL
|
||||
FROM $DISTRO:$REL
|
||||
ARG REL=
|
||||
|
||||
WORKDIR /root
|
||||
|
||||
RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
|
||||
RUN if ([ "${DISTRO}" = "debian" ]) && ( [ "${REL}" = "buster" -o "${REL}" = "bullseye" ] ); then \
|
||||
echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
|
||||
echo >> /etc/apt/preferences; \
|
||||
echo 'Package: *' >> /etc/apt/preferences; \
|
||||
|
@ -23,7 +24,7 @@ RUN apt-get -y build-dep libvirt0
|
|||
RUN apt-get -y install libglusterfs-dev
|
||||
RUN apt-get --download-only source libvirt
|
||||
|
||||
ADD patches/libvirt-5.0-vitastor.diff patches/libvirt-7.0-vitastor.diff patches/libvirt-7.5-vitastor.diff patches/libvirt-7.6-vitastor.diff /root
|
||||
ADD patches/libvirt-5.0-vitastor.diff patches/libvirt-7.0-vitastor.diff patches/libvirt-7.5-vitastor.diff patches/libvirt-7.6-vitastor.diff patches/libvirt-8.0-vitastor.diff /root
|
||||
RUN set -e; \
|
||||
mkdir -p /root/packages/libvirt-$REL; \
|
||||
rm -rf /root/packages/libvirt-$REL/*; \
|
||||
|
|
|
@ -3,4 +3,6 @@ usr/bin/vitastor-cli
|
|||
usr/bin/vitastor-rm
|
||||
usr/bin/vitastor-nbd
|
||||
usr/bin/vitastor-nfs
|
||||
usr/bin/vitastor-kv
|
||||
usr/bin/vitastor-kv-stress
|
||||
usr/lib/*/libvitastor*.so*
|
||||
|
|
|
@ -1,8 +1,10 @@
|
|||
# Build Vitastor packages for Debian inside a container
|
||||
# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
|
||||
# cd ..; podman build --build-arg DISTRO=debian --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
|
||||
|
||||
ARG DISTRO=debian
|
||||
ARG REL=
|
||||
FROM debian:$REL
|
||||
FROM $DISTRO:$REL
|
||||
ARG DISTRO=debian
|
||||
ARG REL=
|
||||
|
||||
WORKDIR /root
|
||||
|
@ -35,8 +37,8 @@ RUN set -e -x; \
|
|||
mkdir -p /root/packages/vitastor-$REL; \
|
||||
rm -rf /root/packages/vitastor-$REL/*; \
|
||||
cd /root/packages/vitastor-$REL; \
|
||||
cp -r /root/vitastor vitastor-1.4.7; \
|
||||
cd vitastor-1.4.7; \
|
||||
cp -r /root/vitastor vitastor-1.5.0; \
|
||||
cd vitastor-1.5.0; \
|
||||
ln -s /root/fio-build/fio-*/ ./fio; \
|
||||
FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||
ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
|
||||
|
@ -49,8 +51,8 @@ RUN set -e -x; \
|
|||
rm -rf a b; \
|
||||
echo "dep:fio=$FIO" > debian/fio_version; \
|
||||
cd /root/packages/vitastor-$REL; \
|
||||
tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.7.orig.tar.xz vitastor-1.4.7; \
|
||||
cd vitastor-1.4.7; \
|
||||
tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.5.0.orig.tar.xz vitastor-1.5.0; \
|
||||
cd vitastor-1.5.0; \
|
||||
V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
|
||||
DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
|
||||
DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
|
||||
|
|
|
@ -9,6 +9,8 @@
|
|||
These parameters apply only to Vitastor clients (QEMU, fio, NBD and so on) and
|
||||
affect their interaction with the cluster.
|
||||
|
||||
- [client_retry_interval](#client_retry_interval)
|
||||
- [client_eio_retry_interval](#client_eio_retry_interval)
|
||||
- [client_max_dirty_bytes](#client_max_dirty_bytes)
|
||||
- [client_max_dirty_ops](#client_max_dirty_ops)
|
||||
- [client_enable_writeback](#client_enable_writeback)
|
||||
|
@ -19,6 +21,26 @@ affect their interaction with the cluster.
|
|||
- [nbd_max_devices](#nbd_max_devices)
|
||||
- [nbd_max_part](#nbd_max_part)
|
||||
|
||||
## client_retry_interval
|
||||
|
||||
- Type: milliseconds
|
||||
- Default: 50
|
||||
- Minimum: 10
|
||||
- Can be changed online: yes
|
||||
|
||||
Retry time for I/O requests failed due to inactive PGs or network
|
||||
connectivity errors.
|
||||
|
||||
## client_eio_retry_interval
|
||||
|
||||
- Type: milliseconds
|
||||
- Default: 1000
|
||||
- Can be changed online: yes
|
||||
|
||||
Retry time for I/O requests failed due to data corruption or unfinished
|
||||
EC object deletions (has_incomplete PG state). 0 disables such retries
|
||||
and clients are not blocked and just get EIO error code instead.
|
||||
|
||||
## client_max_dirty_bytes
|
||||
|
||||
- Type: integer
|
||||
|
|
|
@ -9,6 +9,8 @@
|
|||
Данные параметры применяются только к клиентам Vitastor (QEMU, fio, NBD и т.п.) и
|
||||
затрагивают логику их работы с кластером.
|
||||
|
||||
- [client_retry_interval](#client_retry_interval)
|
||||
- [client_eio_retry_interval](#client_eio_retry_interval)
|
||||
- [client_max_dirty_bytes](#client_max_dirty_bytes)
|
||||
- [client_max_dirty_ops](#client_max_dirty_ops)
|
||||
- [client_enable_writeback](#client_enable_writeback)
|
||||
|
@ -19,6 +21,27 @@
|
|||
- [nbd_max_devices](#nbd_max_devices)
|
||||
- [nbd_max_part](#nbd_max_part)
|
||||
|
||||
## client_retry_interval
|
||||
|
||||
- Тип: миллисекунды
|
||||
- Значение по умолчанию: 50
|
||||
- Минимальное значение: 10
|
||||
- Можно менять на лету: да
|
||||
|
||||
Время повтора запросов ввода-вывода, неудачных из-за неактивных PG или
|
||||
ошибок сети.
|
||||
|
||||
## client_eio_retry_interval
|
||||
|
||||
- Тип: миллисекунды
|
||||
- Значение по умолчанию: 1000
|
||||
- Можно менять на лету: да
|
||||
|
||||
Время повтора запросов ввода-вывода, неудачных из-за повреждения данных
|
||||
или незавершённых удалений EC-объектов (состояния PG has_incomplete).
|
||||
0 отключает повторы таких запросов и клиенты не блокируются, а вместо
|
||||
этого просто получают код ошибки EIO.
|
||||
|
||||
## client_max_dirty_bytes
|
||||
|
||||
- Тип: целое число
|
||||
|
|
|
@ -25,12 +25,11 @@ between clients, OSDs and etcd.
|
|||
- [peer_connect_timeout](#peer_connect_timeout)
|
||||
- [osd_idle_timeout](#osd_idle_timeout)
|
||||
- [osd_ping_timeout](#osd_ping_timeout)
|
||||
- [up_wait_retry_interval](#up_wait_retry_interval)
|
||||
- [max_etcd_attempts](#max_etcd_attempts)
|
||||
- [etcd_quick_timeout](#etcd_quick_timeout)
|
||||
- [etcd_slow_timeout](#etcd_slow_timeout)
|
||||
- [etcd_keepalive_timeout](#etcd_keepalive_timeout)
|
||||
- [etcd_ws_keepalive_timeout](#etcd_ws_keepalive_timeout)
|
||||
- [etcd_ws_keepalive_interval](#etcd_ws_keepalive_interval)
|
||||
|
||||
## tcp_header_buffer_size
|
||||
|
||||
|
@ -212,17 +211,6 @@ Maximum time to wait for OSD keepalive responses. If an OSD doesn't respond
|
|||
within this time, the connection to it is dropped and a reconnection attempt
|
||||
is scheduled.
|
||||
|
||||
## up_wait_retry_interval
|
||||
|
||||
- Type: milliseconds
|
||||
- Default: 50
|
||||
- Minimum: 10
|
||||
- Can be changed online: yes
|
||||
|
||||
OSDs respond to clients with a special error code when they receive I/O
|
||||
requests for a PG that's not synchronized and started. This parameter sets
|
||||
the time for the clients to wait before re-attempting such I/O requests.
|
||||
|
||||
## max_etcd_attempts
|
||||
|
||||
- Type: integer
|
||||
|
@ -257,7 +245,7 @@ Timeout for etcd requests which are allowed to wait for some time.
|
|||
Timeout for etcd connection HTTP Keep-Alive. Should be higher than
|
||||
etcd_report_interval to guarantee that keepalive actually works.
|
||||
|
||||
## etcd_ws_keepalive_timeout
|
||||
## etcd_ws_keepalive_interval
|
||||
|
||||
- Type: seconds
|
||||
- Default: 30
|
||||
|
|
|
@ -25,12 +25,11 @@
|
|||
- [peer_connect_timeout](#peer_connect_timeout)
|
||||
- [osd_idle_timeout](#osd_idle_timeout)
|
||||
- [osd_ping_timeout](#osd_ping_timeout)
|
||||
- [up_wait_retry_interval](#up_wait_retry_interval)
|
||||
- [max_etcd_attempts](#max_etcd_attempts)
|
||||
- [etcd_quick_timeout](#etcd_quick_timeout)
|
||||
- [etcd_slow_timeout](#etcd_slow_timeout)
|
||||
- [etcd_keepalive_timeout](#etcd_keepalive_timeout)
|
||||
- [etcd_ws_keepalive_timeout](#etcd_ws_keepalive_timeout)
|
||||
- [etcd_ws_keepalive_interval](#etcd_ws_keepalive_interval)
|
||||
|
||||
## tcp_header_buffer_size
|
||||
|
||||
|
@ -221,19 +220,6 @@ OSD в любом случае согласовывают реальное зн
|
|||
Если OSD не отвечает за это время, соединение отключается и производится
|
||||
повторная попытка соединения.
|
||||
|
||||
## up_wait_retry_interval
|
||||
|
||||
- Тип: миллисекунды
|
||||
- Значение по умолчанию: 50
|
||||
- Минимальное значение: 10
|
||||
- Можно менять на лету: да
|
||||
|
||||
Когда OSD получают от клиентов запросы ввода-вывода, относящиеся к не
|
||||
поднятым на данный момент на них PG, либо к PG в процессе синхронизации,
|
||||
они отвечают клиентам специальным кодом ошибки, означающим, что клиент
|
||||
должен некоторое время подождать перед повторением запроса. Именно это время
|
||||
ожидания задаёт данный параметр.
|
||||
|
||||
## max_etcd_attempts
|
||||
|
||||
- Тип: целое число
|
||||
|
@ -270,7 +256,7 @@ OSD в любом случае согласовывают реальное зн
|
|||
Таймаут для HTTP Keep-Alive в соединениях к etcd. Должен быть больше, чем
|
||||
etcd_report_interval, чтобы keepalive гарантированно работал.
|
||||
|
||||
## etcd_ws_keepalive_timeout
|
||||
## etcd_ws_keepalive_interval
|
||||
|
||||
- Тип: секунды
|
||||
- Значение по умолчанию: 30
|
||||
|
|
|
@ -41,6 +41,7 @@ Parameters:
|
|||
- [osd_tags](#osd_tags)
|
||||
- [primary_affinity_tags](#primary_affinity_tags)
|
||||
- [scrub_interval](#scrub_interval)
|
||||
- [used_for_fs](#used_for_fs)
|
||||
|
||||
Examples:
|
||||
|
||||
|
@ -154,8 +155,25 @@ That is, if it becomes impossible to place PG data on at least (pg_minsize)
|
|||
OSDs, PG is deactivated for both read and write. So you know that a fresh
|
||||
write always goes to at least (pg_minsize) OSDs (disks).
|
||||
|
||||
That is, pg_size minus pg_minsize sets the number of disk failures to tolerate
|
||||
without temporary downtime (for [osd_out_time](monitor.en.md#osd_out_time)).
|
||||
For example, the difference between pg_minsize 2 and 1 in a 3-way replicated
|
||||
pool (pg_size=3) is:
|
||||
- If 2 hosts go down with pg_minsize=2, the pool becomes inactive and remains
|
||||
inactive for [osd_out_time](monitor.en.md#osd_out_time) (10 minutes). After
|
||||
this timeout, the monitor selects replacement hosts/OSDs and the pool comes
|
||||
up and starts to heal. Therefore, if you don't have replacement OSDs, i.e.
|
||||
if you only have 3 hosts with OSDs and 2 of them are down, the pool remains
|
||||
inactive until you add or return at least 1 host (or change failure_domain
|
||||
to "osd").
|
||||
- If 2 hosts go down with pg_minsize=1, the pool only experiences a short
|
||||
I/O pause until the monitor notices that OSDs are down (5-10 seconds with
|
||||
the default [etcd_report_interval](osd.en.md#etcd_report_interval)). After
|
||||
this pause, I/O resumes, but new data is temporarily written in only 1 copy.
|
||||
Then, after osd_out_time, the monitor also selects replacement OSDs and the
|
||||
pool starts to heal.
|
||||
|
||||
So, pg_minsize regulates the number of failures that a pool can tolerate
|
||||
without temporary downtime for [osd_out_time](monitor.en.md#osd_out_time),
|
||||
but at a cost of slightly reduced storage reliability.
|
||||
|
||||
FIXME: pg_minsize behaviour may be changed in the future to only make PGs
|
||||
read-only instead of deactivating them.
|
||||
|
@ -168,8 +186,8 @@ read-only instead of deactivating them.
|
|||
Number of PGs for this pool. The value should be big enough for the monitor /
|
||||
LP solver to be able to optimize data placement.
|
||||
|
||||
"Enough" is usually around 64-128 PGs per OSD, i.e. you set pg_count for pool
|
||||
to (total OSD count * 100 / pg_size). You can round it to the closest power of 2,
|
||||
"Enough" is usually around 10-100 PGs per OSD, i.e. you set pg_count for pool
|
||||
to (total OSD count * 10 / pg_size). You can round it to the closest power of 2,
|
||||
because it makes it easier to reduce or increase PG count later by dividing or
|
||||
multiplying it by 2.
|
||||
|
||||
|
@ -282,6 +300,25 @@ of the OSDs containing a data chunk for a PG.
|
|||
Automatic scrubbing interval for this pool. Overrides
|
||||
[global scrub_interval setting](osd.en.md#scrub_interval).
|
||||
|
||||
## used_for_fs
|
||||
|
||||
- Type: string
|
||||
|
||||
If non-empty, the pool is marked as used for VitastorFS with metadata stored
|
||||
in block image (regular Vitastor volume) named as the value of this pool parameter.
|
||||
|
||||
When a pool is marked as used for VitastorFS, regular block volume creation in it
|
||||
is disabled (vitastor-cli refuses to create images without --force) to protect
|
||||
the user from block volume and FS file ID collisions and data loss.
|
||||
|
||||
[vitastor-nfs](../usage/nfs.ru.md), in its turn, refuses to use pools not marked
|
||||
for the corresponding FS when starting. This also implies that you can use one
|
||||
pool only for one VitastorFS.
|
||||
|
||||
The second thing that is disabled for VitastorFS pools is reporting per-inode space
|
||||
usage statistics in etcd because a FS pool may store a very large number of files
|
||||
and statistics for them all would take a lot of space in etcd.
|
||||
|
||||
# Examples
|
||||
|
||||
## Replicated pool
|
||||
|
|
|
@ -40,6 +40,7 @@
|
|||
- [osd_tags](#osd_tags)
|
||||
- [primary_affinity_tags](#primary_affinity_tags)
|
||||
- [scrub_interval](#scrub_interval)
|
||||
- [used_for_fs](#used_for_fs)
|
||||
|
||||
Примеры:
|
||||
|
||||
|
@ -157,9 +158,25 @@
|
|||
OSD, PG деактивируется на чтение и запись. Иными словами, всегда известно,
|
||||
что новые блоки данных всегда записываются как минимум на pg_minsize дисков.
|
||||
|
||||
По сути, разница pg_size и pg_minsize задаёт число отказов дисков, которые пул
|
||||
может пережить без временной (на [osd_out_time](monitor.ru.md#osd_out_time))
|
||||
остановки обслуживания.
|
||||
Для примера, разница между pg_minsize 2 и 1 в реплицированном пуле с 3 копиями
|
||||
данных (pg_size=3), проявляется следующим образом:
|
||||
- Если 2 сервера отключаются при pg_minsize=2, пул становится неактивным и
|
||||
остаётся неактивным в течение [osd_out_time](monitor.en.md#osd_out_time)
|
||||
(10 минут), после чего монитор назначает другие OSD/серверы на замену, пул
|
||||
поднимается и начинает восстанавливать недостающие копии данных. Соответственно,
|
||||
если OSD на замену нет - то есть, если у вас всего 3 сервера с OSD и 2 из них
|
||||
недоступны - пул так и остаётся недоступным до тех пор, пока вы не вернёте
|
||||
или не добавите хотя бы 1 сервер (или не переключите failure_domain на "osd").
|
||||
- Если 2 сервера отключаются при pg_minsize=1, ввод-вывод лишь приостанавливается
|
||||
на короткое время, до тех пор, пока монитор не поймёт, что OSD отключены
|
||||
(что занимает 5-10 секунд при стандартном [etcd_report_interval](osd.en.md#etcd_report_interval)).
|
||||
После этого ввод-вывод восстанавливается, но новые данные временно пишутся
|
||||
всего в 1 копии. Когда же проходит osd_out_time, монитор точно так же назначает
|
||||
другие OSD на замену выбывшим и пул начинает восстанавливать копии данных.
|
||||
|
||||
То есть, pg_minsize регулирует число отказов, которые пул может пережить без
|
||||
временной остановки обслуживания на [osd_out_time](monitor.ru.md#osd_out_time),
|
||||
но ценой немного пониженных гарантий надёжности.
|
||||
|
||||
FIXME: Поведение pg_minsize может быть изменено в будущем с полной деактивации
|
||||
PG на перевод их в режим только для чтения.
|
||||
|
@ -172,8 +189,8 @@ PG на перевод их в режим только для чтения.
|
|||
Число PG для данного пула. Число должно быть достаточно большим, чтобы монитор
|
||||
мог равномерно распределить по ним данные.
|
||||
|
||||
Обычно это означает примерно 64-128 PG на 1 OSD, т.е. pg_count можно устанавливать
|
||||
равным (общему числу OSD * 100 / pg_size). Значение можно округлить до ближайшей
|
||||
Обычно это означает примерно 10-100 PG на 1 OSD, т.е. pg_count можно устанавливать
|
||||
равным (общему числу OSD * 10 / pg_size). Значение можно округлить до ближайшей
|
||||
степени 2, чтобы потом было легче уменьшать или увеличивать число PG, умножая
|
||||
или деля его на 2.
|
||||
|
||||
|
@ -290,6 +307,27 @@ OSD с "all".
|
|||
Интервал скраба, то есть, автоматической фоновой проверки данных для данного пула.
|
||||
Переопределяет [глобальную настройку scrub_interval](osd.ru.md#scrub_interval).
|
||||
|
||||
## used_for_fs
|
||||
|
||||
- Type: string
|
||||
|
||||
Если непусто, пул помечается как используемый для файловой системы VitastorFS с
|
||||
метаданными, хранимыми в блочном образе Vitastor с именем, равным значению
|
||||
этого параметра.
|
||||
|
||||
Когда пул помечается как используемый для VitastorFS, создание обычных блочных
|
||||
образов в нём отключается (vitastor-cli отказывается создавать образы без --force),
|
||||
чтобы защитить пользователя от коллизий ID файлов и блочных образов и, таким
|
||||
образом, от потери данных.
|
||||
|
||||
[vitastor-nfs](../usage/nfs.ru.md), в свою очередь, при запуске отказывается
|
||||
использовать для ФС пулы, не выделенные для неё. Это также означает, что один
|
||||
пул может использоваться только для одной VitastorFS.
|
||||
|
||||
Также для ФС-пулов отключается передача статистики в etcd по отдельным инодам,
|
||||
так как ФС-пул может содержать очень много файлов и статистика по ним всем
|
||||
заняла бы очень много места в etcd.
|
||||
|
||||
# Примеры
|
||||
|
||||
## Реплицированный пул
|
||||
|
|
|
@ -1,3 +1,27 @@
|
|||
- name: client_retry_interval
|
||||
type: ms
|
||||
min: 10
|
||||
default: 50
|
||||
online: true
|
||||
info: |
|
||||
Retry time for I/O requests failed due to inactive PGs or network
|
||||
connectivity errors.
|
||||
info_ru: |
|
||||
Время повтора запросов ввода-вывода, неудачных из-за неактивных PG или
|
||||
ошибок сети.
|
||||
- name: client_eio_retry_interval
|
||||
type: ms
|
||||
default: 1000
|
||||
online: true
|
||||
info: |
|
||||
Retry time for I/O requests failed due to data corruption or unfinished
|
||||
EC object deletions (has_incomplete PG state). 0 disables such retries
|
||||
and clients are not blocked and just get EIO error code instead.
|
||||
info_ru: |
|
||||
Время повтора запросов ввода-вывода, неудачных из-за повреждения данных
|
||||
или незавершённых удалений EC-объектов (состояния PG has_incomplete).
|
||||
0 отключает повторы таких запросов и клиенты не блокируются, а вместо
|
||||
этого просто получают код ошибки EIO.
|
||||
- name: client_max_dirty_bytes
|
||||
type: int
|
||||
default: 33554432
|
||||
|
|
|
@ -56,6 +56,8 @@
|
|||
|
||||
{{../../usage/nfs.en.md}}
|
||||
|
||||
{{../../usage/admin.en.md}}
|
||||
|
||||
## Performance
|
||||
|
||||
{{../../performance/understanding.en.md}}
|
||||
|
@ -64,4 +66,6 @@
|
|||
|
||||
{{../../performance/comparison1.en.md}}
|
||||
|
||||
{{../../performance/bench2.en.md}}
|
||||
|
||||
{{../../intro/author.en.md|indent=1}}
|
||||
|
|
|
@ -56,6 +56,8 @@
|
|||
|
||||
{{../../usage/nfs.ru.md}}
|
||||
|
||||
{{../../usage/admin.ru.md}}
|
||||
|
||||
## Производительность
|
||||
|
||||
{{../../performance/understanding.ru.md}}
|
||||
|
@ -64,4 +66,6 @@
|
|||
|
||||
{{../../performance/comparison1.ru.md}}
|
||||
|
||||
{{../../performance/bench2.ru.md}}
|
||||
|
||||
{{../../intro/author.ru.md|indent=1}}
|
||||
|
|
|
@ -243,21 +243,6 @@
|
|||
Максимальное время ожидания ответа на запрос проверки состояния соединения.
|
||||
Если OSD не отвечает за это время, соединение отключается и производится
|
||||
повторная попытка соединения.
|
||||
- name: up_wait_retry_interval
|
||||
type: ms
|
||||
min: 10
|
||||
default: 50
|
||||
online: true
|
||||
info: |
|
||||
OSDs respond to clients with a special error code when they receive I/O
|
||||
requests for a PG that's not synchronized and started. This parameter sets
|
||||
the time for the clients to wait before re-attempting such I/O requests.
|
||||
info_ru: |
|
||||
Когда OSD получают от клиентов запросы ввода-вывода, относящиеся к не
|
||||
поднятым на данный момент на них PG, либо к PG в процессе синхронизации,
|
||||
они отвечают клиентам специальным кодом ошибки, означающим, что клиент
|
||||
должен некоторое время подождать перед повторением запроса. Именно это время
|
||||
ожидания задаёт данный параметр.
|
||||
- name: max_etcd_attempts
|
||||
type: int
|
||||
default: 5
|
||||
|
@ -295,7 +280,7 @@
|
|||
info_ru: |
|
||||
Таймаут для HTTP Keep-Alive в соединениях к etcd. Должен быть больше, чем
|
||||
etcd_report_interval, чтобы keepalive гарантированно работал.
|
||||
- name: etcd_ws_keepalive_timeout
|
||||
- name: etcd_ws_keepalive_interval
|
||||
type: sec
|
||||
default: 30
|
||||
online: true
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
## Server-side features
|
||||
|
||||
- Basic part: highly-available block storage with symmetric clustering and no SPOF
|
||||
- [Performance](../performance/comparison1.en.md) ;-D
|
||||
- [Performance](../performance/bench2.en.md) ;-D
|
||||
- [Multiple redundancy schemes](../config/pool.en.md#scheme): Replication, XOR n+1, Reed-Solomon erasure codes
|
||||
based on jerasure and ISA-L libraries with any number of data and parity drives in a group
|
||||
- Configuration via simple JSON data structures in etcd (parameters, pools and images)
|
||||
|
@ -33,6 +33,7 @@
|
|||
- [Checksums](../config/layout-osd.en.md#data_csum_type)
|
||||
- [Client write-back cache](../config/client.en.md#client_enable_writeback)
|
||||
- [Intelligent recovery auto-tuning](../config/osd.en.md#recovery_tune_interval)
|
||||
- [Clustered file system](../usage/nfs.en.md#vitastorfs)
|
||||
|
||||
## Plugins and tools
|
||||
|
||||
|
@ -46,13 +47,12 @@
|
|||
- [CSI plugin for Kubernetes](../installation/kubernetes.en.md)
|
||||
- [OpenStack support: Cinder driver, Nova and libvirt patches](../installation/openstack.en.md)
|
||||
- [Proxmox storage plugin and packages](../installation/proxmox.en.md)
|
||||
- [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md)
|
||||
- [Simplified NFS proxy for file-based image access emulation (suitable for VMWare)](../usage/nfs.en.md#pseudo-fs)
|
||||
|
||||
## Roadmap
|
||||
|
||||
The following features are planned for the future:
|
||||
|
||||
- File system
|
||||
- Control plane optimisation
|
||||
- Other administrative tools
|
||||
- Web GUI
|
||||
|
|
|
@ -13,7 +13,7 @@
|
|||
## Серверные функции
|
||||
|
||||
- Базовая часть - надёжное кластерное блочное хранилище без единой точки отказа
|
||||
- [Производительность](../performance/comparison1.ru.md) ;-D
|
||||
- [Производительность](../performance/bench2.ru.md) ;-D
|
||||
- [Несколько схем отказоустойчивости](../config/pool.ru.md#scheme): репликация, XOR n+1 (1 диск чётности), коды коррекции ошибок
|
||||
Рида-Соломона на основе библиотек jerasure и ISA-L с любым числом дисков данных и чётности в группе
|
||||
- Конфигурация через простые человекочитаемые JSON-структуры в etcd
|
||||
|
@ -35,6 +35,7 @@
|
|||
- [Контрольные суммы](../config/layout-osd.ru.md#data_csum_type)
|
||||
- [Буферизация записи на стороне клиента](../config/client.ru.md#client_enable_writeback)
|
||||
- [Интеллектуальная автоподстройка скорости восстановления](../config/osd.ru.md#recovery_tune_interval)
|
||||
- [Кластерная файловая система](../usage/nfs.ru.md#vitastorfs)
|
||||
|
||||
## Драйверы и инструменты
|
||||
|
||||
|
@ -48,11 +49,10 @@
|
|||
- [CSI-плагин для Kubernetes](../installation/kubernetes.ru.md)
|
||||
- [Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt](../installation/openstack.ru.md)
|
||||
- [Плагин для Proxmox](../installation/proxmox.ru.md)
|
||||
- [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md)
|
||||
- [Упрощённая NFS-прокси для эмуляции файлового доступа к образам (подходит для VMWare)](../usage/nfs.ru.md#псевдо-фс)
|
||||
|
||||
## Планы развития
|
||||
|
||||
- Файловая система
|
||||
- Оптимизация слоя управления
|
||||
- Другие инструменты администрирования
|
||||
- Web-интерфейс
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
- [Check cluster status](#check-cluster-status)
|
||||
- [Create an image](#create-an-image)
|
||||
- [Install plugins](#install-plugins)
|
||||
- [Create VitastorFS](#create-vitastorfs)
|
||||
|
||||
## Preparation
|
||||
|
||||
|
@ -75,18 +76,16 @@ On the monitor hosts:
|
|||
|
||||
## Create a pool
|
||||
|
||||
Create pool configuration in etcd:
|
||||
Create a pool using vitastor-cli:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool",
|
||||
"scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'
|
||||
vitastor-cli create-pool testpool --pg_size 2 --pg_count 256
|
||||
```
|
||||
|
||||
For EC pools the configuration should look like the following:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
|
||||
"scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}}'
|
||||
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
||||
```
|
||||
|
||||
After you do this, one of the monitors will configure PGs and OSDs will start them.
|
||||
|
@ -116,3 +115,9 @@ After that, you can [run benchmarks](../usage/fio.en.md) or [start QEMU manually
|
|||
- [Proxmox](../installation/proxmox.en.md)
|
||||
- [OpenStack](../installation/openstack.en.md)
|
||||
- [Kubernetes CSI](../installation/kubernetes.en.md)
|
||||
|
||||
## Create VitastorFS
|
||||
|
||||
If you want to use clustered file system in addition to VM or container images:
|
||||
|
||||
- [Follow the instructions here](../usage/nfs.en.md#vitastorfs)
|
||||
|
|
|
@ -14,6 +14,7 @@
|
|||
- [Проверьте состояние кластера](#проверьте-состояние-кластера)
|
||||
- [Создайте образ](#создайте-образ)
|
||||
- [Установите плагины](#установите-плагины)
|
||||
- [Создайте VitastorFS](#создайте-vitastorfs)
|
||||
|
||||
## Подготовка
|
||||
|
||||
|
@ -77,18 +78,16 @@
|
|||
|
||||
## Создайте пул
|
||||
|
||||
Создайте конфигурацию пула с помощью etcdctl:
|
||||
Создайте пул с помощью vitastor-cli:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool",
|
||||
"scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'
|
||||
vitastor-cli create-pool testpool --pg_size 2 --pg_count 256
|
||||
```
|
||||
|
||||
Для пулов с кодами коррекции ошибок конфигурация должна выглядеть примерно так:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
|
||||
"scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}}'
|
||||
vitastor-cli create-pool testpool --ec 2+2 --pg_count 256
|
||||
```
|
||||
|
||||
После этого один из мониторов должен сконфигурировать PG, а OSD должны запустить их.
|
||||
|
@ -118,3 +117,10 @@ vitastor-cli create -s 10G testimg
|
|||
- [Proxmox](../installation/proxmox.ru.md)
|
||||
- [OpenStack](../installation/openstack.ru.md)
|
||||
- [Kubernetes CSI](../installation/kubernetes.ru.md)
|
||||
|
||||
## Создайте VitastorFS
|
||||
|
||||
Если вы хотите использовать не только блочные образы виртуальных машин или контейнеров,
|
||||
а также кластерную файловую систему, то:
|
||||
|
||||
- [Следуйте инструкциям](../usage/nfs.en.md#vitastorfs)
|
||||
|
|
|
@ -0,0 +1,154 @@
|
|||
[Documentation](../../README.md#documentation) → Performance → Newer benchmark of Vitastor 1.3.1
|
||||
|
||||
-----
|
||||
|
||||
[Читать на русском](bench2.ru.md)
|
||||
|
||||
# Newer benchmark of Vitastor 1.3.1
|
||||
|
||||
- [Test environment](#test-environment)
|
||||
- [Notes](#notes)
|
||||
- [Raw drive performance](#raw-drive-performance)
|
||||
- [2 replicas](#2-replicas)
|
||||
- [3 replicas](#3-replicas)
|
||||
- [EC 2+1](#ec-2-1)
|
||||
|
||||
## Test environment
|
||||
|
||||
Hardware configuration: 3 nodes, each with:
|
||||
- 8x NVMe Samsung PM9A3 1.92 TB
|
||||
- 2x Xeon Gold 6342 (24 cores @ 2.8 GHz)
|
||||
- 256 GB RAM
|
||||
- Dual-port 25 GbE Mellanox ConnectX-4 LX network card with RoCEv2
|
||||
- Connected to 2 Mellanox SN2010 switches with MLAG
|
||||
|
||||
## Notes
|
||||
|
||||
Vitastor version was 1.3.1.
|
||||
|
||||
Tests were ran from the storage nodes - 4 fio clients per each of 3 nodes.
|
||||
|
||||
The same large 3 TB image was tested from all hosts because Vitastor has no
|
||||
performance penalties related to running multiple clients over a single inode.
|
||||
|
||||
CPU power saving was disabled. 4 OSDs were created per each NVMe.
|
||||
Checksums were not enabled. Tests with checksums will be conducted later,
|
||||
along with the newer version of Vitastor, and results will be updated.
|
||||
|
||||
CPU configuration was not optimal because of NUMA. It's better to avoid 2-socket
|
||||
platforms. It was especially noticeable in RDMA tests - in the form of ksoftirqd
|
||||
processes (usually 1 per server) eating 100 % of one CPU core and actual bandwidth
|
||||
of one network port reduced to 3-5 Gbit/s instead of 25 Gbit/s - probably because
|
||||
of RFS (Receive Flow Steering) misses. Many network configurations were tried during
|
||||
tests, but nothing helped to solve this problem, so final tests were conducted with
|
||||
the default settings.
|
||||
|
||||
# Raw drive performance
|
||||
|
||||
- Linear write ~1000-2000 MB/s, depending on current state of the drive's garbage collector
|
||||
- Linear read ~3300 MB/s
|
||||
- T1Q1 random write ~60000 iops (latency ~0.015ms)
|
||||
- T1Q1 random read ~14700 iops (latency ~0.066ms)
|
||||
- T1Q16 random write ~180000 iops
|
||||
- T1Q16 random read ~120000 iops
|
||||
- T1Q32 random write ~180000 iops
|
||||
- T1Q32 random read ~195000 iops
|
||||
- T1Q128 random write ~180000 iops
|
||||
- T1Q128 random read ~195000 iops
|
||||
- T4Q128 random write ~525000 iops
|
||||
- T4Q128 random read ~750000 iops
|
||||
|
||||
These numbers make obvious that results could be much better if a faster network
|
||||
was available, because NVMe drives obviously weren't a bottleneck. For example,
|
||||
theoretical maximum linear read performance for 24 drives could be 79.2 GByte/s,
|
||||
which is 633 Gbit/s. Real Vitastor read speed (both linear and random) was around
|
||||
16 Gbyte/s, which is 130 Gbit/s. It's important to note that it was still much
|
||||
larger than the network bandwidth of one server (50 Gbit/s). This is also correct
|
||||
because tests were conducted from all 3 nodes.
|
||||
|
||||
## 2 replicas
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Linear read (4M T6 Q16) | 13.13 GB/s | 16.25 GB/s |
|
||||
| Linear write (4M T6 Q16) | 8.16 GB/s | 7.88 GB/s |
|
||||
| Read 4k T1 Q1 | 8745 iops | 10252 iops |
|
||||
| Write 4k T1 Q1 | 8097 iops | 11488 iops |
|
||||
| Read 4k T12 Q128 | 1305936 iops | 4265861 iops |
|
||||
| Write 4k T12 Q128 | 660490 iops | 1384033 iops |
|
||||
|
||||
CPU consumption OSD per 1 disk:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|---------|---------|
|
||||
| Linear read (4M T6 Q16) | 29.7 % | 29.8 % |
|
||||
| Linear write (4M T6 Q16) | 84.4 % | 33.2 % |
|
||||
| Read 4k T12 Q128 | 98.4 % | 119.1 % |
|
||||
| Write 4k T12 Q128 | 173.4 % | 175.9 % |
|
||||
|
||||
CPU consumption per 1 client (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Linear read (4M T6 Q16) | 100 % | 85.2 % |
|
||||
| Linear write (4M T6 Q16) | 55.8 % | 48.8 % |
|
||||
| Read 4k T12 Q128 | 99.9 % | 96 % |
|
||||
| Write 4k T12 Q128 | 71.6 % | 48.5 % |
|
||||
|
||||
## 3 replicas
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Linear read (4M T6 Q16) | 13.98 GB/s | 16.54 GB/s |
|
||||
| Linear write (4M T6 Q16) | 5.38 GB/s | 5.7 GB/s |
|
||||
| Read 4k T1 Q1 | 8969 iops | 9980 iops |
|
||||
| Write 4k T1 Q1 | 8126 iops | 11672 iops |
|
||||
| Read 4k T12 Q128 | 1358818 iops | 4279088 iops |
|
||||
| Write 4k T12 Q128 | 433890 iops | 993506 iops |
|
||||
|
||||
CPU consumption OSD per 1 disk:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|---------|
|
||||
| Linear read (4M T6 Q16) | 24.9 % | 25.4 % |
|
||||
| Linear write (4M T6 Q16) | 99.3 % | 38.4 % |
|
||||
| Read 4k T12 Q128 | 95.3 % | 111.7 % |
|
||||
| Write 4k T12 Q128 | 173 % | 194 % |
|
||||
|
||||
CPU consumption per 1 client (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Linear read (4M T6 Q16) | 99.9 % | 85.8 % |
|
||||
| Linear write (4M T6 Q16) | 38.9 % | 38.1 % |
|
||||
| Read 4k T12 Q128 | 100 % | 96.1 % |
|
||||
| Write 4k T12 Q128 | 51.6 % | 41.9 % |
|
||||
|
||||
## EC 2+1
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Linear read (4M T6 Q16) | 10.07 GB/s | 11.43 GB/s |
|
||||
| Linear write (4M T6 Q16) | 7.74 GB/s | 8.32 GB/s |
|
||||
| Read 4k T1 Q1 | 7408 iops | 8891 iops |
|
||||
| Write 4k T1 Q1 | 3525 iops | 4903 iops |
|
||||
| Read 4k T12 Q128 | 1216496 iops | 2552765 iops |
|
||||
| Write 4k T12 Q128 | 278110 iops | 821261 iops |
|
||||
|
||||
CPU consumption OSD per 1 disk:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|---------|---------|
|
||||
| Linear read (4M T6 Q16) | 68.6 % | 33.6 % |
|
||||
| Linear write (4M T6 Q16) | 108.3 % | 50.2 % |
|
||||
| Read 4k T12 Q128 | 138.1 % | 97.9 % |
|
||||
| Write 4k T12 Q128 | 168.7 % | 188.5 % |
|
||||
|
||||
CPU consumption per 1 client (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Linear read (4M T6 Q16) | 88.2 % | 52.4 % |
|
||||
| Linear write (4M T6 Q16) | 51.8 % | 46.8 % |
|
||||
| Read 4k T12 Q128 | 99.7 % | 61.3 % |
|
||||
| Write 4k T12 Q128 | 35.1 % | 31.3 % |
|
|
@ -0,0 +1,157 @@
|
|||
[Документация](../../README-ru.md#документация) → Производительность → Более новый тест Vitastor 1.3.1
|
||||
|
||||
-----
|
||||
|
||||
[Read in English](bench2.en.md)
|
||||
|
||||
# Более новый тест Vitastor 1.3.1
|
||||
|
||||
- [Описание стенда](#описание-стенда)
|
||||
- [Примечания](#примечания)
|
||||
- [Производительность голых дисков](#производительность-голых-дисков)
|
||||
- [2 реплики](#2-реплики)
|
||||
- [3 реплики](#3-реплики)
|
||||
- [EC 2+1](#ec-2-1)
|
||||
|
||||
## Описание стенда
|
||||
|
||||
Железо: 3 сервера, в каждом:
|
||||
- 8x NVMe Samsung PM9A3 1.92 TB
|
||||
- 2x Xeon Gold 6342 (24 ядра @ 2.8 GHz)
|
||||
- 256 GB RAM
|
||||
- Двухпортовая 25 GbE сетевая карта Mellanox ConnectX-4 LX с поддержкой RoCEv2
|
||||
- Подключение к 2 коммутаторам Mellanox SN2010 в MLAG
|
||||
|
||||
## Примечания
|
||||
|
||||
Версия Vitastor 1.3.1.
|
||||
|
||||
Тесты проводились с самих серверов хранения - по 4 клиента fio с каждого из 3 серверов.
|
||||
|
||||
Тестировался один большой образ размером 3 ТБ со всех хостов - создавать отдельные образы
|
||||
для тестов в Vitastor необязательно, т.к. в Vitastor нет замедления при записи в один
|
||||
узел несколькими клиентами.
|
||||
|
||||
Экономия энергии CPU отключена. На каждый NVMe создавалось 4 OSD.
|
||||
Контрольные суммы не включались. Тесты с контрольными суммами будут проведены
|
||||
позднее. Тогда же будет протестирована более новая версия Vitastor, и результаты
|
||||
будут обновлены.
|
||||
|
||||
Конфигурация CPU стенда неоптимальна из-за NUMA - двухпроцессорных серверов лучше
|
||||
избегать. Особенно это проявлялось во время тестов с RDMA - выражалось это в потреблении
|
||||
100% одного ядра CPU одним процессом ksoftirqd и работой одного из двух сетевых портов
|
||||
на скорости 3-5 ГБит/с вместо 25 ГБит/с - предположительно, из-за "непопаданий" RFS
|
||||
(Receive Flow Steering) на нужные ядра. Решить эту проблему во время проведения тестов
|
||||
не получилось. Было перепробовано множество различных настроек, но в итоге тесты проведены
|
||||
с настройками по умолчанию, т.к. улучшения добиться не удалось.
|
||||
|
||||
# Производительность голых дисков
|
||||
|
||||
- Линейная запись ~1000-2000 МБ/с, в зависимости от состояния сборщика мусора диска
|
||||
- Линейное чтение ~3300 МБ/с
|
||||
- T1Q1 запись ~60000 iops (задержка ~0.015ms)
|
||||
- T1Q1 чтение ~14700 iops (задержка ~0.066ms)
|
||||
- T1Q16 запись ~180000 iops
|
||||
- T1Q16 чтение ~120000 iops
|
||||
- T1Q32 запись ~180000 iops
|
||||
- T1Q32 чтение ~195000 iops
|
||||
- T1Q128 запись ~180000 iops
|
||||
- T1Q128 чтение ~195000 iops
|
||||
- T4Q128 запись ~525000 iops
|
||||
- T4Q128 чтение ~750000 iops
|
||||
|
||||
Из данных цифр очевидно, что при наличии более быстрой сети результаты были бы
|
||||
значительно лучше, так как в диски тест, очевидно, не упирался. Например, теоретический предел по
|
||||
линейному чтению для 24 таких дисков был бы около 79.2 ГБайт/с, то есть,
|
||||
633 гигабита в секунду. Реальная скорость чтения (и случайного, и линейного)
|
||||
Vitastor составила примерно 16 ГБайт/с, то есть 130 гигабит в секунду. При этом
|
||||
следует заметить, что этот результат всё равно значительно лучше пропускной способности
|
||||
сети отдельно взятого узла - что тоже вполне логично, так как тест выполнялся со
|
||||
всех трёх узлов.
|
||||
|
||||
## 2 реплики
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Линейное чтение (4M T6 Q16) | 13.13 ГБ/с | 16.25 ГБ/с |
|
||||
| Линейная запись (4M T6 Q16) | 8.16 ГБ/с | 7.88 ГБ/с |
|
||||
| Чтение 4k T1 Q1 | 8745 iops | 10252 iops |
|
||||
| Запись 4k T1 Q1 | 8097 iops | 11488 iops |
|
||||
| Чтение 4k T12 Q128 | 1305936 iops | 4265861 iops |
|
||||
| Запись 4k T12 Q128 | 660490 iops | 1384033 iops |
|
||||
|
||||
Потребление CPU OSD на 1 диск:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|---------|---------|
|
||||
| Линейное чтение (4M T6 Q16) | 29.7 % | 29.8 % |
|
||||
| Линейная запись (4M T6 Q16) | 84.4 % | 33.2 % |
|
||||
| Чтение 4k T12 Q128 | 98.4 % | 119.1 % |
|
||||
| Запись 4k T12 Q128 | 173.4 % | 175.9 % |
|
||||
|
||||
Потребление CPU на 1 клиента (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Линейное чтение (4M T6 Q16) | 100 % | 85.2 % |
|
||||
| Линейная запись (4M T6 Q16) | 55.8 % | 48.8 % |
|
||||
| Чтение 4k T12 Q128 | 99.9 % | 96 % |
|
||||
| Запись 4k T12 Q128 | 71.6 % | 48.5 % |
|
||||
|
||||
## 3 реплики
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Линейное чтение (4M T6 Q16) | 13.98 ГБ/с | 16.54 ГБ/с |
|
||||
| Линейная запись (4M T6 Q16) | 5.38 ГБ/с | 5.7 ГБ/с |
|
||||
| Чтение 4k T1 Q1 | 8969 iops | 9980 iops |
|
||||
| Запись 4k T1 Q1 | 8126 iops | 11672 iops |
|
||||
| Чтение 4k T12 Q128 | 1358818 iops | 4279088 iops |
|
||||
| Запись 4k T12 Q128 | 433890 iops | 993506 iops |
|
||||
|
||||
Потребление CPU OSD на 1 диск:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|---------|
|
||||
| Линейное чтение (4M T6 Q16) | 24.9 % | 25.4 % |
|
||||
| Линейная запись (4M T6 Q16) | 99.3 % | 38.4 % |
|
||||
| Чтение 4k T12 Q128 | 95.3 % | 111.7 % |
|
||||
| Запись 4k T12 Q128 | 173 % | 194 % |
|
||||
|
||||
Потребление CPU на 1 клиента (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Линейное чтение (4M T6 Q16) | 99.9 % | 85.8 % |
|
||||
| Линейная запись (4M T6 Q16) | 38.9 % | 38.1 % |
|
||||
| Чтение 4k T12 Q128 | 100 % | 96.1 % |
|
||||
| Запись 4k T12 Q128 | 51.6 % | 41.9 % |
|
||||
|
||||
## EC 2+1
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------------|--------------|
|
||||
| Линейное чтение (4M T6 Q16) | 10.07 ГБ/с | 11.43 ГБ/с |
|
||||
| Линейная запись (4M T6 Q16) | 7.74 ГБ/с | 8.32 ГБ/с |
|
||||
| Чтение 4k T1 Q1 | 7408 iops | 8891 iops |
|
||||
| Запись 4k T1 Q1 | 3525 iops | 4903 iops |
|
||||
| Чтение 4k T12 Q128 | 1216496 iops | 2552765 iops |
|
||||
| Запись 4k T12 Q128 | 278110 iops | 821261 iops |
|
||||
|
||||
Потребление CPU OSD на 1 диск:
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|---------|---------|
|
||||
| Линейное чтение (4M T6 Q16) | 68.6 % | 33.6 % |
|
||||
| Линейная запись (4M T6 Q16) | 108.3 % | 50.2 % |
|
||||
| Чтение 4k T12 Q128 | 138.1 % | 97.9 % |
|
||||
| Запись 4k T12 Q128 | 168.7 % | 188.5 % |
|
||||
|
||||
Потребление CPU на 1 клиента (fio):
|
||||
|
||||
| | TCP | RDMA |
|
||||
|------------------------------|--------|--------|
|
||||
| Линейное чтение (4M T6 Q16) | 88.2 % | 52.4 % |
|
||||
| Линейная запись (4M T6 Q16) | 51.8 % | 46.8 % |
|
||||
| Чтение 4k T12 Q128 | 99.7 % | 61.3 % |
|
||||
| Запись 4k T12 Q128 | 35.1 % | 31.3 % |
|
|
@ -0,0 +1,215 @@
|
|||
[Documentation](../../README.md#documentation) → Usage → Administration
|
||||
|
||||
-----
|
||||
|
||||
[Читать на русском](admin.ru.md)
|
||||
|
||||
# Administration
|
||||
|
||||
- [Pool states](#pool-states)
|
||||
- [PG states](#pg-states)
|
||||
- [Base PG states](#base-pg-states)
|
||||
- [Additional PG states](#additional-pg-states)
|
||||
- [Removing a healthy disk](#removing-a-healthy-disk)
|
||||
- [Removing a failed disk](#removing-a-failed-disk)
|
||||
- [Adding a disk](#adding-a-disk)
|
||||
- [Restoring from lost pool configuration](#restoring-from-lost-pool-configuration)
|
||||
- [Upgrading Vitastor](#upgrading-vitastor)
|
||||
- [OSD memory usage](#osd-memory-usage)
|
||||
|
||||
## Pool states
|
||||
|
||||
Pool is active — that is, fully available for client input/output — when all its PGs are
|
||||
'active' (maybe with some additional state flags).
|
||||
|
||||
If at least 1 PG is inactive, pool is also inactive and all clients suspend their I/O and
|
||||
wait until you fix the cluster. :-)
|
||||
|
||||
## PG states
|
||||
|
||||
PG states may be seen in [vitastor-cli status](cli.en.md#status) output.
|
||||
|
||||
PG state consists of exactly 1 base state and an arbitrary number of additional states.
|
||||
|
||||
### Base PG states
|
||||
|
||||
PG state always includes exactly 1 of the following base states:
|
||||
- **active** — PG is active and handles user I/O.
|
||||
- **incomplete** — Not enough OSDs are available to activate this PG. That is, more disks
|
||||
are lost than it's allowed by the pool's redundancy scheme. For example, if the pool has
|
||||
pg_size=3 and pg_minsize=1, part of the data may be written only to 1 OSD. If that exact
|
||||
OSD is lost, PG will become **incomplete**.
|
||||
- **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for
|
||||
this PG at all (if the pool is just created), or an unavailable OSD is set as primary,
|
||||
or the primary OSD refuses to start this PG (for example, because of wrong block_size),
|
||||
or the PG is stopped by the monitor using `pause: true` flag in `/vitastor/config/pgs` in etcd.
|
||||
- **starting** — primary OSD has acquired PG lock in etcd, PG is starting.
|
||||
- **peering** — primary OSD requests PG object listings from secondary OSDs and calculates
|
||||
the PG state.
|
||||
- **repeering** — PG is waiting for current I/O operations to complete and will
|
||||
then transition to **peering**.
|
||||
- **stopping** — PG is waiting for current I/O operations to complete and will
|
||||
then transition to **offline** or be activated by another OSD.
|
||||
|
||||
All states except **active** mean that PG is inactive and client I/O is suspended.
|
||||
|
||||
**peering** state is normally visible only for a short period of time during OSD restarts
|
||||
and during switching primary OSD of PGs.
|
||||
|
||||
**starting**, **repeering**, **stopping** states normally almost aren't visible at all.
|
||||
If you notice them for any noticeable time — chances are some operations on some OSDs hung.
|
||||
Search for "slow op" in OSD logs to find them — operations hung for more than
|
||||
[slow_log_interval](../config/osd.en.md#slow_log_interval) are logged as "slow ops".
|
||||
|
||||
State transition diagram:
|
||||
|
||||
![PG state transitions](pg_states.svg "PG state transitions")
|
||||
|
||||
### Additional PG states
|
||||
|
||||
If a PG is active it can also have any number of the following additional states:
|
||||
|
||||
- **degraded** — PG is running on reduced number of drives (OSDs), redundancy of all
|
||||
objects in this PG is reduced.
|
||||
- **has_incomplete** — some objects in this PG are incomplete (unrecoverable), that is,
|
||||
they have too many lost EC parts (more than pool's [parity_chunks](../config/pool.en.md#parity_chunks)).
|
||||
- **has_degraded** — some objects in this PG have reduced redundancy
|
||||
compared to the rest of the PG (so PG can be degraded+has_degraded at the same time).
|
||||
These objects should be healed automatically by recovery process, unless
|
||||
it's disabled by [no_recovery](../config/osd.en.md#no_recovery).
|
||||
- **has_misplaced** — some objects in this PG are stored on an OSD set different from
|
||||
the target set of the PG. These objects should be moved automatically, unless
|
||||
rebalance is disabled by [no_rebalance](../config/osd.en.md#no_rebalance). Objects
|
||||
that are degraded and misplaced at the same time are treated as just degraded.
|
||||
- **has_unclean** — one more state normally noticeable only for very short time during
|
||||
PG activation. It's used only with EC pools and means that some objects of this PG
|
||||
have started but not finished modifications. All such objects are either quickly
|
||||
committed or rolled back by the primary OSD when starting the PG, that is why the
|
||||
state shouldn't be noticeable. If you notice it, it probably means that commit or
|
||||
rollback operations are hung.
|
||||
- **has_invalid** — PG contains objects with incorrect part ID. Never occurs normally.
|
||||
It can only occur if you delete a non-empty EC pool and then recreate it as a replica
|
||||
pool or with smaller data part count.
|
||||
- **has_corrupted** — PG has corrupted objects, discovered by checking checksums during
|
||||
read or during scrub. When possible, such objects should be recovered automatically.
|
||||
If objects remain corrupted, use [vitastor-cli describe](cli.en.md#describe) to find
|
||||
out details and/or look into the log of the primary OSD of the PG.
|
||||
- **has_inconsistent** — PG has objects with non-matching parts or copies on different OSDs,
|
||||
and it's impossible to determine which copy is correct automatically. It may happen
|
||||
if you use a pool with 2 replica and you don't enable checksums, and if data on one
|
||||
of replicas becomes corrupted. You should also use vitastor-cli [describe](cli.en.md#describe)
|
||||
and [fix](cli.en.md#fix) commands to remove the incorrect version in this case.
|
||||
- **left_on_dead** — part of the data of this PG is left on unavailable OSD that isn't
|
||||
fully removed from the cluster. You should either start the corresponding OSD back and
|
||||
let it remove the unneeded data or remove it from cluster using vitastor-cli
|
||||
[rm-osd](cli.en.md#rm-osd) if you know that it's gone forever (for example, if the disk died).
|
||||
- **scrubbing** — data [scrub](../config/osd.en.md#auto_scrub) is running for this PG.
|
||||
|
||||
## Removing a healthy disk
|
||||
|
||||
Befor removing a healthy disk from the cluster set its OSD weight(s) to 0 to
|
||||
move data away. To do that, add `"reweight":0` to etcd key `/vitastor/config/osd/<OSD_NUMBER>`.
|
||||
For example:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
||||
```
|
||||
|
||||
Then wait until rebalance finishes and remove OSD by running `vitastor-disk purge /dev/vitastor/osdN-data`.
|
||||
|
||||
## Removing a failed disk
|
||||
|
||||
If a disk is already dead, its OSD(s) are likely already stopped.
|
||||
|
||||
In this case just remove OSD(s) from the cluster by running `vitastor-cli rm-osd OSD_NUMBER`.
|
||||
|
||||
## Adding a disk
|
||||
|
||||
If you're adding a server, first install Vitastor packages and copy the
|
||||
`/etc/vitastor/vitastor.conf` configuration file to it.
|
||||
|
||||
After that you can just run `vitastor-disk prepare /dev/nvmeXXX`, of course with
|
||||
the same parameters which you used for other OSDs in your cluster before.
|
||||
|
||||
## Restoring from lost pool configuration
|
||||
|
||||
If you remove or corrupt `/vitastor/config/pools` key in etcd all pools will
|
||||
be deleted. Don't worry, the data won't be lost, but you'll need to perform
|
||||
a specific recovery procedure.
|
||||
|
||||
First you need to restore previous configuration of the pool with the same ID
|
||||
and EC/replica parameters and wait until pool PGs appear in `vitastor-cli status`.
|
||||
|
||||
Then add all OSDs into the history records of all PGs. You can do it by running
|
||||
the following script (just don't forget to use your own PG_COUNT and POOL_ID):
|
||||
|
||||
```
|
||||
PG_COUNT=32
|
||||
POOL_ID=1
|
||||
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
||||
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
||||
for i in $(seq 1 $PG_COUNT); do
|
||||
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'; done
|
||||
done
|
||||
```
|
||||
|
||||
After that all PGs should peer and find all previous data.
|
||||
|
||||
## Upgrading Vitastor
|
||||
|
||||
Every upcoming Vitastor version is usually compatible with previous both forward
|
||||
and backward regarding the network protocol and etcd data structures.
|
||||
|
||||
So, by default, if this page doesn't contain explicit different instructions, you
|
||||
can upgrade your Vitastor cluster by simply upgrading packages and restarting all
|
||||
OSDs and monitors in any order.
|
||||
|
||||
Upgrading is performed without stopping clients (VMs/containers), you just need to
|
||||
upgrade and restart servers one by one. However, ideally you should restart VMs too
|
||||
to make them use the new version of the client library.
|
||||
|
||||
Exceptions (specific upgrade instructions):
|
||||
- Upgrading <= 1.1.x to 1.2.0 or later, if you use EC n+k with k>=2, is recommended
|
||||
to be performed with full downtime: first you should stop all clients, then all OSDs,
|
||||
then upgrade and start everything back — because versions before 1.2.0 have several
|
||||
bugs leading to invalid data being read in EC n+k, k>=2 configurations in degraded pools.
|
||||
- Versions <= 0.8.7 are incompatible with versions >= 0.9.0, so you should first
|
||||
upgrade from <= 0.8.7 to 0.8.8 or 0.8.9, and only then to >= 0.9.x. If you upgrade
|
||||
without this intermediate step, client I/O will hang until the end of upgrade process.
|
||||
- Upgrading from <= 0.5.x to >= 0.6.x is not supported.
|
||||
|
||||
Rollback:
|
||||
- Version 1.0.0 has a new disk format, so OSDs initiaziled on 1.0.0 can't be rolled
|
||||
back to 0.9.x or previous versions.
|
||||
- Versions before 0.8.0 don't have vitastor-disk, so OSDs, initialized by it, won't
|
||||
start with 0.7.x or 0.6.x. :-)
|
||||
|
||||
## OSD memory usage
|
||||
|
||||
OSD uses RAM mainly for:
|
||||
|
||||
- Metadata index: `data_size`/[`block_size`](../config/layout-cluster.en.md#block_size) * `approximately 1.1` * `32` bytes.
|
||||
Consumed always.
|
||||
- Copy of the on-disk metadata area: `data_size`/[`block_size`](../config/layout-cluster.en.md#block_size) * `28` bytes.
|
||||
Consumed if [inmemory_metadata](../config/osd.en.md#inmemory_metadata) isn't disabled.
|
||||
- Bitmaps: `data_size`/[`bitmap_granularity`](../config/layout-cluster.en.md#bitmap_granularity)/`8` * `2` bytes.
|
||||
Consumed always.
|
||||
- Journal index: between 0 and, approximately, journal size. Consumed always.
|
||||
- Copy of the on-disk journal area: exactly journal size. Consumed if
|
||||
[inmemory_journal](../config/osd.en.md#inmemory_journal) isn't disabled.
|
||||
- Checksums: `data_size`/[`csum_block_size`](../config/osd.en.md#csum_block_size) * 4 bytes.
|
||||
Consumed if checksums are enabled and [inmemory_metadata](../config/osd.en.md#inmemory_metadata) isn't disabled.
|
||||
|
||||
bitmap_granularity is almost always 4 KB.
|
||||
|
||||
So with default SSD settings (block_size=128k, journal_size=32M, csum_block_size=4k) memory usage is:
|
||||
|
||||
- Metadata and bitmaps: ~600 MB per 1 TB of data.
|
||||
- Journal: up to 64 MB per 1 OSD.
|
||||
- Checksums: 1 GB per 1 TB of data.
|
||||
|
||||
With default HDD settings (block_size=1M, journal_size=128M, csum_block_size=32k):
|
||||
|
||||
- Metadata and bitmaps: ~128 MB per 1 TB of data.
|
||||
- Journal: up to 256 MB per 1 OSD.
|
||||
- Checksums: 128 MB per 1 TB of data.
|
|
@ -0,0 +1,211 @@
|
|||
[Документация](../../README-ru.md#документация) → Использование → Администрирование
|
||||
|
||||
-----
|
||||
|
||||
[Read in English](admin.en.md)
|
||||
|
||||
# Администрирование
|
||||
|
||||
- [Состояния пулов](#состояния-пулов)
|
||||
- [Состояния PG](#состояния-pg)
|
||||
- [Базовые состояния PG](#базовые-состояния-pg)
|
||||
- [Дополнительные состояния PG](#дополнительные-состояния-pg)
|
||||
- [Удаление исправного диска](#удаление-исправного-диска)
|
||||
- [Удаление неисправного диска](#удаление-неисправного-диска)
|
||||
- [Добавление диска](#добавление-диска)
|
||||
- [Восстановление потерянной конфигурации пулов](#восстановление-потерянной-конфигурации-пулов)
|
||||
- [Обновление Vitastor](#обновление-vitastor)
|
||||
- [Потребление памяти OSD](#потребление-памяти-osd)
|
||||
|
||||
## Состояния пулов
|
||||
|
||||
Пул активен — то есть, полностью доступен для клиентского ввода-вывода — когда все его PG
|
||||
активны, то есть, имеют статус active, возможно, с любым набором дополнительных флагов.
|
||||
|
||||
Если хотя бы 1 PG неактивна, пул неактивен и все клиенты зависают и ждут, пока вы почините
|
||||
кластер. :-)
|
||||
|
||||
## Состояния PG
|
||||
|
||||
Вы можете видеть состояния PG в выводе команды [vitastor-cli status](cli.ru.md#status).
|
||||
|
||||
Состояние PG состоит из ровно 1 базового флага состояния, плюс любого числа дополнительных.
|
||||
|
||||
### Базовые состояния PG
|
||||
|
||||
Состояние PG включает в себя ровно 1 флаг из следующих:
|
||||
- **active** — PG активна и обрабатывает запросы ввода-вывода от пользователей.
|
||||
- **incomplete** — Недостаточно живых OSD, чтобы включить эту PG.
|
||||
То есть, дисков потеряно больше, чем разрешено схемой отказоустойчивости пула и pg_minsize.
|
||||
Например, если у пула pg_size=3 и pg_minsize=1, то часть данных может записаться всего на 1 OSD.
|
||||
Если потом конкретно этот OSD упадёт, PG окажется **incomplete**.
|
||||
- **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще
|
||||
(если пул только создан), либо в качестве первичного назначен недоступный OSD, либо
|
||||
назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size),
|
||||
либо PG остановлена монитором через флаг `pause: true` в `/vitastor/config/pgs` в etcd.
|
||||
- **starting** — первичный OSD захватил блокировку PG в etcd, PG запускается.
|
||||
- **peering** — первичный OSD опрашивает вторичные OSD на предмет списков объектов данной PG и рассчитывает её состояние.
|
||||
- **repeering** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **peering**.
|
||||
- **stopping** — PG ожидает завершения текущих операций ввода-вывода, после чего перейдёт в состояние **offline** или поднимется на другом OSD.
|
||||
|
||||
Все состояния, кроме **active**, означают, что PG неактивна и ввод-вывод приостановлен.
|
||||
|
||||
Состояние **peering** в норме заметно только при перезапуске OSD или переключении первичных
|
||||
OSD, на протяжении небольшого периода времени.
|
||||
|
||||
Состояния **starting**, **repeering**, **stopping** в норме практически не заметны вообще,
|
||||
PG должны очень быстро переходить из них в другие. Если эти состояния заметны
|
||||
хоть сколько-то значительное время — вероятно, какие-то операции на каких-то OSD зависли.
|
||||
Чтобы найти их, ищите "slow op" в журналах OSD — операции, зависшие дольше,
|
||||
чем на [slow_log_interval](../config/osd.ru.md#slow_log_interval), записываются в
|
||||
журналы OSD как "slow op".
|
||||
|
||||
Диаграмма переходов:
|
||||
|
||||
![Диаграмма переходов](pg_states.svg "Диаграмма переходов")
|
||||
|
||||
### Дополнительные состояния PG
|
||||
|
||||
Если PG активна, она также может иметь любое число дополнительных флагов состояний:
|
||||
|
||||
- **degraded** — PG поднята на неполном числе дисков (OSD), избыточность хранения всех объектов снижена.
|
||||
- **has_incomplete** — часть объектов в PG неполные (невосстановимые), то есть, у них потеряно
|
||||
слишком много EC-частей (больше, чем [parity_chunks](../config/pool.ru.md#parity_chunks) пула).
|
||||
- **has_degraded** — часть объектов в PG деградированы, избыточность их хранения снижена по сравнению
|
||||
с остальным содержимым данной PG (то есть, PG может одновременно быть degraded+has_degraded).
|
||||
Данные объекты должны восстановиться автоматически, если только восстановление не отключено
|
||||
через [no_recovery](../config/osd.ru.md#no_recovery).
|
||||
- **has_misplaced** — часть объектов в PG сейчас расположена не на целевом наборе OSD этой PG.
|
||||
Данные объекты должны переместиться автоматически, если только перебалансировка не отключена
|
||||
через [no_rebalance](../config/osd.ru.md#no_rebalance). Объекты, являющиеся одновременно
|
||||
degraded и misplaced, считаются просто degraded.
|
||||
- **has_unclean** — ещё одно состояние, в норме заметное только очень короткое время при поднятии PG.
|
||||
Применяется только к EC и означает, что на каких-то OSD этой PG есть EC-части объектов, для которых
|
||||
был начат, но не завершён процесс записи. Все такие объекты первичный OSD либо завершает, либо
|
||||
откатывает при поднятии PG первым делом, поэтому состояние и не должно быть заметно. Опять-таки,
|
||||
если оно заметно — значит, скорее всего, операции отката или завершения записи на каких-то OSD зависли.
|
||||
- **has_invalid** — в PG найдены объекты с некорректными ID части. В норме не проявляется вообще
|
||||
никогда, проявляется только если, не удалив данные, создать на месте EC-пула либо реплика-пул,
|
||||
либо EC-пул с меньшим числом частей данных.
|
||||
- **has_corrupted** — в PG есть повреждённые объекты, обнаруженные с помощью контрольных сумм или
|
||||
скраба (сверки копий). Если объекты можно восстановить, они восстановятся автоматически. Если
|
||||
не восстанавливаются, используйте команду [vitastor-cli describe](cli.ru.md#describe) для
|
||||
выяснения деталей и/или смотрите в журнал первичного OSD данной PG.
|
||||
- **has_inconsistent** — в PG есть объекты, у которых не совпадают копии/части данных на разных OSD,
|
||||
и при этом автоматически определить, какая копия верная, а какая нет, невозможно. Такое может
|
||||
произойти, если вы используете 2 реплики, не включали контрольные суммы, и на одной из реплик
|
||||
данные повредились. В этом случае тоже надо использовать команды vitastor-cli [describe](cli.ru.md#describe)
|
||||
и [fix](cli.ru.md#fix) для удаления некорректной версии.
|
||||
- **left_on_dead** — часть данных PG осталась на отключённом, но не удалённом из кластера окончательно,
|
||||
OSD. Вам нужно либо вернуть соответствующий OSD в строй и дать ему очистить лишние данные, либо
|
||||
удалить его из кластера окончательно с помощью vitastor-cli [rm-osd](cli.ru.md#rm-osd), если
|
||||
известно, что он уже не вернётся (например, если умер диск).
|
||||
- **scrubbing** — идёт фоновая проверка данных PG ([скраб](../config/osd.ru.md#auto_scrub)).
|
||||
|
||||
## Удаление исправного диска
|
||||
|
||||
Перед удалением исправного диска из кластера установите его OSD вес в 0, чтобы убрать с него данные.
|
||||
Для этого добавьте в ключ `/vitastor/config/osd/<НОМЕР_OSD>` в etcd значение `"reweight":0`, например:
|
||||
|
||||
```
|
||||
etcdctl --endpoints=http://1.1.1.1:2379/v3 put /vitastor/config/osd/1 '{"reweight":0}'
|
||||
```
|
||||
|
||||
Дождитесь завершения ребаланса, после чего удалите OSD командой `vitastor-disk purge /dev/vitastor/osdN-data`.
|
||||
|
||||
## Удаление неисправного диска
|
||||
|
||||
Если диск уже умер, его OSD, скорее всего, уже будет/будут остановлен(ы).
|
||||
|
||||
В этом случае просто удалите OSD из etcd командой `vitastor-cli rm-osd НОМЕР_OSD`.
|
||||
|
||||
## Добавление диска
|
||||
|
||||
Если сервер новый, установите на него пакеты Vitastor и скопируйте файл конфигурации
|
||||
`/etc/vitastor/vitastor.conf`.
|
||||
|
||||
После этого достаточно выполнить команду `vitastor-disk prepare /dev/nvmeXXX`, разумеется,
|
||||
с параметрами, аналогичными другим OSD в вашем кластере.
|
||||
|
||||
## Восстановление потерянной конфигурации пулов
|
||||
|
||||
Если удалить или повредить ключ `/vitastor/config/pools` в etcd, все пулы будут удалены.
|
||||
Не волнуйтесь, данные потеряны не будут, но вам нужно будет провести специальную
|
||||
процедуру восстановления.
|
||||
|
||||
Сначала нужно будет восстановить конфигурацию пулов, создав пул с таким же ID и
|
||||
с такими же параметрами EC/реплик, и подождать, пока PG пула появятся в `vitastor-cli status`.
|
||||
|
||||
Далее нужно будет добавить все OSD в исторические записи всех PG. Примерно так
|
||||
(только подставьте свои PG_COUNT и POOL_ID):
|
||||
|
||||
```
|
||||
PG_COUNT=32
|
||||
POOL_ID=1
|
||||
ALL_OSDS=$(etcdctl --endpoints=your_etcd_address:2379 get --keys-only --prefix /vitastor/osd/stats/ | \
|
||||
perl -e '$/ = undef; $a = <>; $a =~ s/\s*$//; $a =~ s!/vitastor/osd/stats/!!g; $a =~ s/\s+/,/g; print $a')
|
||||
for i in $(seq 1 $PG_COUNT); do
|
||||
etcdctl --endpoints=your_etcd_address:2379 put /vitastor/pg/history/$POOL_ID/$i '{"all_peers":['$ALL_OSDS']}'; done
|
||||
done
|
||||
```
|
||||
|
||||
После этого все PG должны пройти peering и найти все предыдущие данные.
|
||||
|
||||
## Обновление Vitastor
|
||||
|
||||
Обычно каждая следующая версия Vitastor совместима с предыдущими и "вперёд", и "назад"
|
||||
с точки зрения сетевого протокола и структур данных в etcd.
|
||||
|
||||
Так что по умолчанию, если на данной странице не указано обратное, считается, что для
|
||||
обновления достаточно обновить пакеты и перезапустить все OSD и мониторы Vitastor в
|
||||
произвольном порядке.
|
||||
|
||||
Обновление производится без остановки клиентов (виртуальных машин/контейнеров), для этого
|
||||
достаточно обновлять серверы по одному. Однако, конечно, чтобы запущенные виртуальные машины
|
||||
начали использовать новую версию клиентской библиотеки, их тоже нужно перезапустить.
|
||||
|
||||
Исключения (особые указания при обновлении):
|
||||
- Обновляться с версий <= 1.1.x до версий >= 1.2.0, если вы используете EC n+k и k>=2,
|
||||
рекомендуется с временной остановкой кластера — сначала нужно остановить всех клиентов,
|
||||
потом все OSD, потом обновить и запустить всё обратно — из-за нескольких багов, которые
|
||||
могли приводить к некорректному чтению данных в деградированных EC-пулах.
|
||||
- Версии <= 0.8.7 несовместимы с версиями >= 0.9.0, поэтому при обновлении с <= 0.8.7
|
||||
нужно сначала обновиться до 0.8.8 или 0.8.9, а уже потом до любых версий >= 0.9.x.
|
||||
Иначе клиентский ввод-вывод зависнет до завершения обновления.
|
||||
- Обновление с версий 0.5.x и более ранних до 0.6.x и более поздних не поддерживается.
|
||||
|
||||
Откат:
|
||||
- В версии 1.0.0 поменялся дисковый формат, поэтому OSD, созданные на версии >= 1.0.0,
|
||||
нельзя откатить до версии 0.9.x и более ранних.
|
||||
- В версиях ранее 0.8.0 нет vitastor-disk, значит, созданные им OSD нельзя откатить
|
||||
до 0.7.x или 0.6.x. :-)
|
||||
|
||||
## Потребление памяти OSD
|
||||
|
||||
Основное потребление памяти складывается из:
|
||||
|
||||
- Индекс метаданных: `размер_данных`/[`block_size`](../config/layout-cluster.ru.md#block_size) * `примерно 1.1` * `32` байт.
|
||||
Потребляется всегда.
|
||||
- Копия дисковой области метаданных: `размер_данных`/[`block_size`](../config/layout-cluster.ru.md#block_size) * `28` байт.
|
||||
Потребляется, если не отключена настройка [inmemory_metadata](../config/osd.ru.md#inmemory_metadata).
|
||||
- Битмапы: `размер_данных`/[`bitmap_granularity`](../config/layout-cluster.ru.md#bitmap_granularity)/`8` * `2` байт.
|
||||
Потребляется всегда.
|
||||
- Индекс журнала: от 0 до, приблизительно, размера журнала. Потребляется всегда.
|
||||
- Копия дисковой области журнала: в точности размер журнала. Потребляется,
|
||||
если не отключена настройка [inmemory_journal](../config/osd.ru.md#inmemory_journal).
|
||||
- Контрольные суммы: `размер_данных`/[`csum_block_size`](../config/osd.ru.md#csum_block_size) * `4` байт.
|
||||
Потребляется, если включены контрольные суммы и не отключена настройка [inmemory_metadata](../config/osd.ru.md#inmemory_metadata).
|
||||
|
||||
bitmap_granularity, как правило, никогда не меняется и равен 4 килобайтам.
|
||||
|
||||
Таким образом, при SSD-настройках по умолчанию (block_size=128k, journal_size=32M, csum_block_size=4k) потребляется:
|
||||
|
||||
- Метаданные и битмапы: ~600 МБ на 1 ТБ данных
|
||||
- Журнал: до 64 МБ на 1 OSD
|
||||
- Контрольные суммы: 1 ГБ на 1 ТБ данных
|
||||
|
||||
При HDD-настройках по умолчанию (block_size=1M, journal_size=128M, csum_block_size=32k):
|
||||
|
||||
- Метаданные и битмапы: ~128 МБ на 1 ТБ данных
|
||||
- Журнал: до 256 МБ на 1 OSD
|
||||
- Контрольные суммы: 128 МБ на 1 ТБ данных
|
|
@ -24,6 +24,10 @@ It supports the following commands:
|
|||
- [fix](#fix)
|
||||
- [alloc-osd](#alloc-osd)
|
||||
- [rm-osd](#rm-osd)
|
||||
- [create-pool](#create-pool)
|
||||
- [modify-pool](#modify-pool)
|
||||
- [ls-pools](#ls-pools)
|
||||
- [rm-pool](#rm-pool)
|
||||
|
||||
Global options:
|
||||
|
||||
|
@ -131,19 +135,18 @@ See also about [how to export snapshots](qemu.en.md#exporting-snapshots).
|
|||
|
||||
## modify
|
||||
|
||||
`vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force]`
|
||||
`vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force] [--down-ok]`
|
||||
|
||||
Rename, resize image or change its readonly status. Images with children can't be made read-write.
|
||||
If the new size is smaller than the old size, extra data will be purged.
|
||||
You should resize file system in the image, if present, before shrinking it.
|
||||
|
||||
```
|
||||
-f|--force Proceed with shrinking or setting readwrite flag even if the image has children.
|
||||
```
|
||||
* `-f|--force` - Proceed with shrinking or setting readwrite flag even if the image has children.
|
||||
* `--down-ok` - Proceed with shrinking even if some data will be left on unavailable OSDs.
|
||||
|
||||
## rm
|
||||
|
||||
`vitastor-cli rm <from> [<to>] [--writers-stopped]`
|
||||
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
||||
|
||||
Remove `<from>` or all layers between `<from>` and `<to>` (`<to>` must be a child of `<from>`),
|
||||
rebasing all their children accordingly. --writers-stopped allows merging to be a bit
|
||||
|
@ -151,6 +154,10 @@ more effective in case of a single 'slim' read-write child and 'fat' removed par
|
|||
the child is merged into parent and parent is renamed to child in that case.
|
||||
In other cases parent layers are always merged into children.
|
||||
|
||||
Other options:
|
||||
|
||||
* `--down-ok` - Continue deletion/merging even if some data will be left on unavailable OSDs.
|
||||
|
||||
## flatten
|
||||
|
||||
`vitastor-cli flatten <layer>`
|
||||
|
@ -238,3 +245,91 @@ Refuses to remove OSDs with data without `--force` and `--allow-data-loss`.
|
|||
|
||||
With `--dry-run` only checks if deletion is possible without data loss and
|
||||
redundancy degradation.
|
||||
|
||||
## create-pool
|
||||
|
||||
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
||||
|
||||
Create a pool. Required parameters:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|--------------------------|---------------------------------------------------------------------------------------|
|
||||
| `-s R` or `--pg_size R` | Number of replicas for replicated pools |
|
||||
| `--ec N+K` | Number of data (N) and parity (K) chunks for erasure-coded pools |
|
||||
| `-n N` or `--pg_count N` | PG count for the new pool (start with 10*<OSD count>/pg_size rounded to a power of 2) |
|
||||
|
||||
Optional parameters:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|--------------------------------|----------------------------------------------------------------------------|
|
||||
| `--pg_minsize <number>` | R or N+K minus number of failures to tolerate without downtime ([details](../config/pool.en.md#pg_minsize)) |
|
||||
| `--failure_domain host` | Failure domain: host, osd or a level from placement_levels. Default: host |
|
||||
| `--root_node <node>` | Put pool only on child OSDs of this placement tree node |
|
||||
| `--osd_tags <tag>[,<tag>]...` | Put pool only on OSDs tagged with all specified tags |
|
||||
| `--block_size 128k` | Put pool only on OSDs with this data block size |
|
||||
| `--bitmap_granularity 4k` | Put pool only on OSDs with this logical sector size |
|
||||
| `--immediate_commit none` | Put pool only on OSDs with this or larger immediate_commit (none < small < all) |
|
||||
| `--primary_affinity_tags tags` | Prefer to put primary copies on OSDs with all specified tags |
|
||||
| `--scrub_interval <time>` | Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y |
|
||||
| `--used_for_fs <name>` | Mark pool as used for VitastorFS with metadata in image <name> |
|
||||
| `--pg_stripe_size <number>` | Increase object grouping stripe |
|
||||
| `--max_osd_combinations 10000` | Maximum number of random combinations for LP solver input |
|
||||
| `--wait` | Wait for the new pool to come online |
|
||||
| `-f` or `--force` | Do not check that cluster has enough OSDs to create the pool |
|
||||
|
||||
See also [Pool configuration](../config/pool.en.md) for detailed parameter descriptions.
|
||||
|
||||
Examples:
|
||||
|
||||
`vitastor-cli create-pool test_x4 -s 4 -n 32`
|
||||
|
||||
`vitastor-cli create-pool test_ec42 --ec 4+2 -n 32`
|
||||
|
||||
## modify-pool
|
||||
|
||||
`vitastor-cli modify-pool|pool-modify <id|name> [--name <new_name>] [PARAMETERS...]`
|
||||
|
||||
Modify an existing pool. Modifiable parameters:
|
||||
|
||||
```
|
||||
[-s|--pg_size <number>] [--pg_minsize <number>] [-n|--pg_count <count>]
|
||||
[--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>] [--no_inode_stats 0|1]
|
||||
[--max_osd_combinations <number>] [--primary_affinity_tags <tags>] [--scrub_interval <time>]
|
||||
```
|
||||
|
||||
Non-modifiable parameters (changing them WILL lead to data loss):
|
||||
|
||||
```
|
||||
[--block_size <size>] [--bitmap_granularity <size>]
|
||||
[--immediate_commit <all|small|none>] [--pg_stripe_size <size>]
|
||||
```
|
||||
|
||||
These, however, can still be modified with -f|--force.
|
||||
|
||||
See [create-pool](#create-pool) for parameter descriptions.
|
||||
|
||||
Examples:
|
||||
|
||||
`vitastor-cli modify-pool pool_A --name pool_B`
|
||||
|
||||
`vitastor-cli modify-pool 2 --pg_size 4 -n 128`
|
||||
|
||||
## rm-pool
|
||||
|
||||
`vitastor-cli rm-pool|pool-rm [--force] <id|name>`
|
||||
|
||||
Remove a pool. Refuses to remove pools with images without `--force`.
|
||||
|
||||
## ls-pools
|
||||
|
||||
`vitastor-cli ls-pools|pool-ls|ls-pool|pools [-l] [--detail] [--sort FIELD] [-r] [-n N] [--stats] [<glob> ...]`
|
||||
|
||||
List pools (only matching <glob> patterns if passed).
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|----------------------|-------------------------------------------------------|
|
||||
| `-l` or `--long` | Also report I/O statistics |
|
||||
| `--detail` | Use list format (not table), show all details |
|
||||
| `--sort FIELD` | Sort by specified field (see fields in --json output) |
|
||||
| `-r` or `--reverse` | Sort in descending order |
|
||||
| `-n` or `--count N` | Only list first N items |
|
||||
|
|
|
@ -23,6 +23,10 @@ vitastor-cli - интерфейс командной строки для адм
|
|||
- [merge-data](#merge-data)
|
||||
- [alloc-osd](#alloc-osd)
|
||||
- [rm-osd](#rm-osd)
|
||||
- [create-pool](#create-pool)
|
||||
- [modify-pool](#modify-pool)
|
||||
- [ls-pools](#ls-pools)
|
||||
- [rm-pool](#rm-pool)
|
||||
|
||||
Глобальные опции:
|
||||
|
||||
|
@ -85,8 +89,8 @@ kaveri 2/1 32 0 B 10 G 0 B 100% 0%
|
|||
|
||||
`vitastor-cli ls [-l] [-p POOL] [--sort FIELD] [-r] [-n N] [<glob> ...]`
|
||||
|
||||
Показать список образов, если переданы шаблоны `<glob>`, то только с именами,
|
||||
соответствующими этим шаблонам (стандартные ФС-шаблоны с * и ?).
|
||||
Показать список образов, если передан(ы) шаблон(ы) `<glob>`, то только с именами,
|
||||
соответствующими одному из шаблонов (стандартные ФС-шаблоны с * и ?).
|
||||
|
||||
Опции:
|
||||
|
||||
|
@ -132,7 +136,7 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
|
|||
|
||||
## modify
|
||||
|
||||
`vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force]`
|
||||
`vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force] [--down-ok]`
|
||||
|
||||
Изменить размер, имя образа или флаг "только для чтения". Снимать флаг "только для чтения"
|
||||
и уменьшать размер образов, у которых есть дочерние клоны, без `--force` нельзя.
|
||||
|
@ -140,13 +144,12 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
|
|||
Если новый размер меньше старого, "лишние" данные будут удалены, поэтому перед уменьшением
|
||||
образа сначала уменьшите файловую систему в нём.
|
||||
|
||||
```
|
||||
-f|--force Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
|
||||
```
|
||||
* `-f|--force` - Разрешить уменьшение или перевод в чтение-запись образа, у которого есть клоны.
|
||||
* `--down-ok` - Разрешить уменьшение, даже если часть данных останется неудалённой на недоступных OSD.
|
||||
|
||||
## rm
|
||||
|
||||
`vitastor-cli rm <from> [<to>] [--writers-stopped]`
|
||||
`vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]`
|
||||
|
||||
Удалить образ `<from>` или все слои от `<from>` до `<to>` (`<to>` должен быть дочерним
|
||||
образом `<from>`), одновременно меняя родительские образы их клонов (если таковые есть).
|
||||
|
@ -158,6 +161,10 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
|
|||
|
||||
В других случаях родительские слои вливаются в дочерние.
|
||||
|
||||
Другие опции:
|
||||
|
||||
* `--down-ok` - Продолжать удаление/слияние, даже если часть данных останется неудалённой на недоступных OSD.
|
||||
|
||||
## flatten
|
||||
|
||||
`vitastor-cli flatten <layer>`
|
||||
|
@ -255,3 +262,91 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
|
|||
|
||||
С опцией `--dry-run` только проверяет, возможно ли удаление без потери данных и деградации
|
||||
избыточности.
|
||||
|
||||
## create-pool
|
||||
|
||||
`vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]`
|
||||
|
||||
Создать пул. Обязательные параметры:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|---------------------------|---------------------------------------------------------------------------------------------|
|
||||
| `-s R` или `--pg_size R` | Число копий данных для реплицированных пулов |
|
||||
| `--ec N+K` | Число частей данных (N) и чётности (K) для пулов с кодами коррекции ошибок |
|
||||
| `-n N` или `--pg_count N` | Число PG для нового пула (начните с 10*<число OSD>/pg_size, округлённого до степени двойки) |
|
||||
|
||||
Необязательные параметры:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|--------------------------------|----------------------------------------------------------------------------|
|
||||
| `--pg_minsize <number>` | (R или N+K) минус число разрешённых отказов без остановки пула ([подробнее](../config/pool.ru.md#pg_minsize)) |
|
||||
| `--failure_domain host` | Домен отказа: host, osd или другой из placement_levels. По умолчанию: host |
|
||||
| `--root_node <node>` | Использовать для пула только дочерние OSD этого узла дерева размещения |
|
||||
| `--osd_tags <tag>[,<tag>]...` | ...только OSD со всеми заданными тегами |
|
||||
| `--block_size 128k` | ...только OSD с данным размером блока |
|
||||
| `--bitmap_granularity 4k` | ...только OSD с данным размером логического сектора |
|
||||
| `--immediate_commit none` | ...только OSD с этим или большим immediate_commit (none < small < all) |
|
||||
| `--primary_affinity_tags tags` | Предпочитать OSD со всеми данными тегами для роли первичных |
|
||||
| `--scrub_interval <time>` | Включить скрабы с заданным интервалом времени (число + единица s/m/h/d/M/y) |
|
||||
| `--pg_stripe_size <number>` | Увеличить блок группировки объектов по PG |
|
||||
| `--max_osd_combinations 10000` | Максимальное число случайных комбинаций OSD для ЛП-солвера |
|
||||
| `--wait` | Подождать, пока новый пул будет активирован |
|
||||
| `-f` или `--force` | Не проверять, что в кластере достаточно доменов отказа для создания пула |
|
||||
|
||||
Подробно о параметрах см. [Конфигурация пулов](../config/pool.ru.md).
|
||||
|
||||
Примеры:
|
||||
|
||||
`vitastor-cli create-pool test_x4 -s 4 -n 32`
|
||||
|
||||
`vitastor-cli create-pool test_ec42 --ec 4+2 -n 32`
|
||||
|
||||
## modify-pool
|
||||
|
||||
`vitastor-cli modify-pool|pool-modify <id|name> [--name <new_name>] [PARAMETERS...]`
|
||||
|
||||
Изменить настройки существующего пула. Изменяемые параметры:
|
||||
|
||||
```
|
||||
[-s|--pg_size <number>] [--pg_minsize <number>] [-n|--pg_count <count>]
|
||||
[--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>]
|
||||
[--max_osd_combinations <number>] [--primary_affinity_tags <tags>] [--scrub_interval <time>]
|
||||
```
|
||||
|
||||
Неизменяемые параметры (их изменение ПРИВЕДЁТ к потере данных):
|
||||
|
||||
```
|
||||
[--block_size <size>] [--bitmap_granularity <size>]
|
||||
[--immediate_commit <all|small|none>] [--pg_stripe_size <size>]
|
||||
```
|
||||
|
||||
Эти параметры можно изменить, только если явно передать опцию -f или --force.
|
||||
|
||||
Описания параметров смотрите в [create-pool](#create-pool).
|
||||
|
||||
Примеры:
|
||||
|
||||
`vitastor-cli modify-pool pool_A --name pool_B`
|
||||
|
||||
`vitastor-cli modify-pool 2 --pg_size 4 -n 128`
|
||||
|
||||
## rm-pool
|
||||
|
||||
`vitastor-cli rm-pool|pool-rm [--force] <id|name>`
|
||||
|
||||
Удалить пул. Отказывается удалять пул, в котором ещё есть образы, без `--force`.
|
||||
|
||||
## ls-pools
|
||||
|
||||
`vitastor-cli ls-pools|pool-ls|ls-pool|pools [-l] [--detail] [--sort FIELD] [-r] [-n N] [--stats] [<glob> ...]`
|
||||
|
||||
Показать список пулов. Если передан(ы) шаблон(ы) `<glob>`, то только с именами,
|
||||
соответствующими одному из шаблонов (стандартные ФС-шаблоны с * и ?).
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|-----------------------|------------------------------------------------------------|
|
||||
| `-l` или `--long` | Вывести также статистику ввода-вывода |
|
||||
| `--detail` | Максимально подробный вывод в виде списка (а не таблицы) |
|
||||
| `--sort FIELD` | Сортировать по заданному полю (поля см. в выводе с --json) |
|
||||
| `-r` или `--reverse` | Сортировать в обратном порядке |
|
||||
| `-n` или `--count N` | Выводить только первые N записей |
|
||||
|
|
|
@ -88,7 +88,7 @@ Options (both modes):
|
|||
--block_size 1M/128k Set blockstore object size
|
||||
--bitmap_granularity 4k Set bitmap granularity
|
||||
--data_csum_type none Set data checksum type (crc32c or none)
|
||||
--csum_block_size 4k Set data checksum block size
|
||||
--csum_block_size 4k/32k Set data checksum block size (SSD/HDD default)
|
||||
--data_device_block 4k Override data device block size
|
||||
--meta_device_block 4k Override metadata device block size
|
||||
--journal_device_block 4k Override journal device block size
|
||||
|
|
|
@ -89,7 +89,7 @@ vitastor-disk - инструмент командной строки для уп
|
|||
--block_size 1M/128k Задать размер объекта хранилища
|
||||
--bitmap_granularity 4k Задать гранулярность битовых карт
|
||||
--data_csum_type none Задать тип контрольных сумм (crc32c или none)
|
||||
--csum_block_size 4k Задать размер блока расчёта контрольных сумм
|
||||
--csum_block_size 4k/32k Задать размер блока расчёта контрольных сумм (дефолт SSD/HDD)
|
||||
--data_device_block 4k Задать размер блока устройства данных
|
||||
--meta_device_block 4k Задать размер блока метаданных
|
||||
--journal_device_block 4k Задать размер блока журнала
|
||||
|
|
|
@ -1,45 +1,153 @@
|
|||
[Documentation](../../README.md#documentation) → Usage → NFS
|
||||
[Documentation](../../README.md#documentation) → Usage → VitastorFS and pseudo-FS
|
||||
|
||||
-----
|
||||
|
||||
[Читать на русском](nfs.ru.md)
|
||||
|
||||
# NFS
|
||||
# VitastorFS and pseudo-FS
|
||||
|
||||
Vitastor has a simplified NFS 3.0 proxy for file-based image access emulation. It's not
|
||||
suitable as a full-featured file system, at least because all file/image metadata is stored
|
||||
in etcd and kept in memory all the time - thus you can't put a lot of files in it.
|
||||
Vitastor has two file system implementations. Both can be used via `vitastor-nfs`.
|
||||
|
||||
However, NFS proxy is totally fine as a method to provide VM image access and allows to
|
||||
plug Vitastor into, for example, VMWare. It's important to note that for VMWare it's a much
|
||||
better access method than iSCSI, because with iSCSI we'd have to put all VM images into one
|
||||
Vitastor image exported as a LUN to VMWare and formatted with VMFS. VMWare doesn't use VMFS
|
||||
over NFS.
|
||||
Commands:
|
||||
- [mount](#mount)
|
||||
- [start](#start)
|
||||
|
||||
NFS proxy is stateless if you use immediate_commit=all mode (for SSD with capacitors or
|
||||
HDDs with disabled cache), so you can run multiple NFS proxies and use a network load
|
||||
balancer or any failover method you want to in that case.
|
||||
## Pseudo-FS
|
||||
|
||||
vitastor-nfs usage:
|
||||
Simplified pseudo-FS proxy is used for file-based image access emulation. It's not
|
||||
suitable as a full-featured file system: it lacks a lot of FS features, it stores
|
||||
all file/image metadata in memory and in etcd. So it's fine for hundreds or thousands
|
||||
of large files/images, but not for millions.
|
||||
|
||||
Pseudo-FS proxy is intended for environments where other block volume access methods
|
||||
can't be used or impose additional restrictions - for example, VMWare. NFS is better
|
||||
for VMWare than, for example, iSCSI, because with iSCSI, VMWare puts all VM images
|
||||
into one large shared block image in its own VMFS file system, and with NFS, VMWare
|
||||
doesn't use VMFS and puts each VM disk in a regular file which is equal to one
|
||||
Vitastor block image, just as originally intended.
|
||||
|
||||
To use Vitastor pseudo-FS locally, run `vitastor-nfs mount --block /mnt/vita`.
|
||||
|
||||
Also you can start the network server:
|
||||
|
||||
```
|
||||
vitastor-nfs [STANDARD OPTIONS] [OTHER OPTIONS]
|
||||
|
||||
--subdir <DIR> export images prefixed <DIR>/ (default empty - export all images)
|
||||
--portmap 0 do not listen on port 111 (portmap/rpcbind, requires root)
|
||||
--bind <IP> bind service to <IP> address (default 0.0.0.0)
|
||||
--nfspath <PATH> set NFS export path to <PATH> (default is /)
|
||||
--port <PORT> use port <PORT> for NFS services (default is 2049)
|
||||
--pool <POOL> use <POOL> as default pool for new files (images)
|
||||
--foreground 1 stay in foreground, do not daemonize
|
||||
vitastor-nfs start --block --etcd_address 192.168.5.10:2379 --portmap 0 --port 2050 --pool testpool
|
||||
```
|
||||
|
||||
Example start and mount commands (etcd_address is optional):
|
||||
To mount the FS exported by this server, run:
|
||||
|
||||
```
|
||||
vitastor-nfs --etcd_address 192.168.5.10:2379 --portmap 0 --port 2050 --pool testpool
|
||||
mount server:/ /mnt/ -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
|
||||
```
|
||||
|
||||
```
|
||||
mount localhost:/ /mnt/ -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
|
||||
```
|
||||
## VitastorFS
|
||||
|
||||
VitastorFS is a full-featured clustered (Read-Write-Many) file system. It supports most POSIX
|
||||
features like hierarchical organization, symbolic links, hard links, quick renames and so on.
|
||||
|
||||
VitastorFS metadata is stored in a Parallel Optimistic B-Tree key-value database,
|
||||
implemented over a regular Vitastor block volume. Directory entries and inodes
|
||||
are stored in a simple human-readable JSON format in the B-Tree. `vitastor-kv` tool
|
||||
can be used to inspect the database.
|
||||
|
||||
To use VitastorFS:
|
||||
|
||||
1. Create a pool or choose an existing empty pool for FS data
|
||||
2. Create an image for FS metadata, preferably in a faster (SSD or replica-HDD) pool,
|
||||
but you can create it in the data pool too if you want (image size doesn't matter):
|
||||
`vitastor-cli create -s 10G -p fastpool testfs`
|
||||
3. Mark data pool as an FS pool: `vitastor-cli modify-pool --used-for-fs testfs data-pool`
|
||||
4. Either mount the FS: `vitastor-nfs mount --fs testfs --pool data-pool /mnt/vita`
|
||||
5. Or start the NFS server: `vitastor-nfs start --fs testfs --pool data-pool`
|
||||
|
||||
### Supported POSIX features
|
||||
|
||||
- Read-after-write semantics (read returns new data immediately after write)
|
||||
- Linear and random read and write
|
||||
- Writing outside current file size
|
||||
- Hierarchical structure, immediate rename of files and directories
|
||||
- File size change support (truncate)
|
||||
- Permissions (chmod/chown)
|
||||
- Flushing data to stable storage (if required) (fsync)
|
||||
- Symbolic links
|
||||
- Hard links
|
||||
- Special files (devices, sockets, named pipes)
|
||||
- File modification and attribute change time tracking (mtime and ctime)
|
||||
- Modification time (mtime) and last access time (atime) change support (utimes)
|
||||
- Correct handling of directory listing during file creation/deletion
|
||||
|
||||
### Limitations
|
||||
|
||||
POSIX features currently not implemented in VitastorFS:
|
||||
- File locking is not supported
|
||||
- Actually used space is not counted, so `du` always reports apparent file sizes
|
||||
instead of actually allocated space
|
||||
- Access times (`atime`) are not tracked (like `-o noatime`)
|
||||
- Modification time (`mtime`) is updated lazily every second (like `-o lazytime`)
|
||||
|
||||
Other notable missing features which should be addressed in the future:
|
||||
- Defragmentation of "shared" inodes. Files smaller than pool object size (block_size
|
||||
multiplied by data part count if pool is EC) are internally stored in large block
|
||||
volumes sequentially, one after another, and leave garbage after deleting or resizing.
|
||||
Defragmentator will be implemented to collect this garbage.
|
||||
- Inode ID reuse. Currently inode IDs always grow, the limit is 2^48 inodes, so
|
||||
in theory you may hit it if you create and delete a very large number of files
|
||||
- Compaction of the key-value B-Tree. Current implementation never merges or deletes
|
||||
B-Tree blocks, so B-Tree may become bloated over time. Currently you can
|
||||
use `vitastor-kv dumpjson` & `loadjson` commands to recreate the index in such
|
||||
situations.
|
||||
- Filesystem check tool. VitastorFS doesn't have journal because it would impose a
|
||||
severe performance hit, optimistic CAS-based transactions are used instead of it.
|
||||
So, again, in theory an abnormal shutdown of the FS server may leave some garbage
|
||||
in the DB. The FS is implemented is such way that this garbage doesn't affect its
|
||||
function, but having a tool to clean it up still seems a right thing to do.
|
||||
|
||||
## Horizontal scaling
|
||||
|
||||
Linux NFS 3.0 client doesn't support built-in scaling or failover, i.e. you can't
|
||||
specify multiple server addresses when mounting the FS.
|
||||
|
||||
However, you can use any regular TCP load balancing over multiple NFS servers.
|
||||
It's absolutely safe with `immediate_commit=all` and `client_enable_writeback=false`
|
||||
settings, because Vitastor NFS proxy doesn't keep uncommitted data in memory
|
||||
with these settings. But it may even work without `immediate_commit=all` because
|
||||
the Linux NFS client repeats all uncommitted writes if it loses the connection.
|
||||
|
||||
## Commands
|
||||
|
||||
### mount
|
||||
|
||||
`vitastor-nfs (--fs <NAME> | --block) [-o <OPT>] mount <MOUNTPOINT>`
|
||||
|
||||
Start local filesystem server and mount file system to <MOUNTPOINT>.
|
||||
|
||||
Use regular `umount <MOUNTPOINT>` to unmount the FS.
|
||||
|
||||
The server will be automatically stopped when the FS is unmounted.
|
||||
|
||||
- `-o|--options <OPT>` - Pass additional NFS mount options (ex.: -o async).
|
||||
|
||||
### start
|
||||
|
||||
`vitastor-nfs (--fs <NAME> | --block) start`
|
||||
|
||||
Start network NFS server. Options:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|-----------------|------------------------------------------------------------|
|
||||
| `--bind <IP>` | bind service to \<IP> address (default 0.0.0.0) |
|
||||
| `--port <PORT>` | use port \<PORT> for NFS services (default is 2049) |
|
||||
| `--portmap 0` | do not listen on port 111 (portmap/rpcbind, requires root) |
|
||||
|
||||
## Common options
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|--------------------|----------------------------------------------------------|
|
||||
| `--fs <NAME>` | use VitastorFS with metadata in image \<NAME> |
|
||||
| `--block` | use pseudo-FS presenting images as files |
|
||||
| `--pool <POOL>` | use \<POOL> as default pool for new files |
|
||||
| `--subdir <DIR>` | export \<DIR> instead of root directory (pseudo-FS only) |
|
||||
| `--nfspath <PATH>` | set NFS export path to \<PATH> (default is /) |
|
||||
| `--pidfile <FILE>` | write process ID to the specified file |
|
||||
| `--logfile <FILE>` | log to the specified file |
|
||||
| `--foreground 1` | stay in foreground, do not daemonize |
|
||||
|
|
|
@ -1,44 +1,159 @@
|
|||
[Документация](../../README-ru.md#документация) → Использование → NFS
|
||||
[Документация](../../README-ru.md#документация) → Использование → VitastorFS и псевдо-ФС
|
||||
|
||||
-----
|
||||
|
||||
[Read in English](nfs.en.md)
|
||||
|
||||
# NFS
|
||||
# VitastorFS и псевдо-ФС
|
||||
|
||||
В Vitastor реализована упрощённая NFS 3.0 прокси для эмуляции файлового доступа к образам.
|
||||
Это не полноценная файловая система, т.к. метаданные всех файлов (образов) сохраняются
|
||||
в etcd и всё время хранятся в оперативной памяти - то есть, положить туда много файлов
|
||||
не получится.
|
||||
В Vitastor есть две реализации файловой системы. Обе используются через `vitastor-nfs`.
|
||||
|
||||
Однако в качестве способа доступа к образам виртуальных машин NFS прокси прекрасно подходит
|
||||
и позволяет подключить Vitastor, например, к VMWare.
|
||||
Команды:
|
||||
- [mount](#mount)
|
||||
- [start](#start)
|
||||
|
||||
При этом, если вы используете режим immediate_commit=all (для SSD с конденсаторами или HDD
|
||||
с отключённым кэшем), то NFS-сервер не имеет состояния и вы можете свободно поднять
|
||||
его в нескольких экземплярах и использовать поверх них сетевой балансировщик нагрузки или
|
||||
схему с отказоустойчивостью.
|
||||
## Псевдо-ФС
|
||||
|
||||
Использование vitastor-nfs:
|
||||
Упрощённая реализация псевдо-ФС используется для эмуляции файлового доступа к блочным
|
||||
образам Vitastor. Это не полноценная файловая система - в ней отсутствуют многие функции
|
||||
POSIX ФС, а метаданные всех файлов (образов) сохраняются в etcd и всё время хранятся в
|
||||
оперативной памяти - то есть, псевдо-ФС подходит для сотен или тысяч файлов, но не миллионов.
|
||||
|
||||
Псевдо-ФС предназначена для доступа к образам виртуальных машин в средах, где другие
|
||||
способы невозможны или неудобны - например, в VMWare. Для VMWare это лучшая опция, чем
|
||||
iSCSI, так как при использовании iSCSI VMWare размещает все виртуальные машины в одном
|
||||
большом блочном образе внутри собственной ФС VMFS, а с NFS VMFS не используется и каждый
|
||||
диск ВМ представляется в виде одного файла, то есть, соответствует одному блочному образу
|
||||
Vitastor, как это и задумано изначально.
|
||||
|
||||
Чтобы подключить псевдо-ФС Vitastor, выполните команду `vitastor-nfs mount --block /mnt/vita`.
|
||||
|
||||
Либо же запустите сетевой вариант сервера:
|
||||
|
||||
```
|
||||
vitastor-nfs [СТАНДАРТНЫЕ ОПЦИИ] [ДРУГИЕ ОПЦИИ]
|
||||
|
||||
--subdir <DIR> экспортировать "поддиректорию" - образы с префиксом имени <DIR>/ (по умолчанию пусто - экспортировать все образы)
|
||||
--portmap 0 отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий)
|
||||
--bind <IP> принимать соединения по адресу <IP> (по умолчанию 0.0.0.0 - на всех)
|
||||
--nfspath <PATH> установить путь NFS-экспорта в <PATH> (по умолчанию /)
|
||||
--port <PORT> использовать порт <PORT> для NFS-сервисов (по умолчанию 2049)
|
||||
--pool <POOL> использовать пул <POOL> для новых образов (обязательно, если пул в кластере не один)
|
||||
--foreground 1 не уходить в фон после запуска
|
||||
vitastor-nfs start --block --etcd_address 192.168.5.10:2379 --portmap 0 --port 2050 --pool testpool
|
||||
```
|
||||
|
||||
Пример монтирования Vitastor через NFS (etcd_address необязателен):
|
||||
Примонтировать ФС, запущенную с такими опциями, можно следующей командой:
|
||||
|
||||
```
|
||||
vitastor-nfs --etcd_address 192.168.5.10:2379 --portmap 0 --port 2050 --pool testpool
|
||||
mount server:/ /mnt/ -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
|
||||
```
|
||||
|
||||
```
|
||||
mount localhost:/ /mnt/ -o port=2050,mountport=2050,nfsvers=3,soft,nolock,tcp
|
||||
```
|
||||
## VitastorFS
|
||||
|
||||
VitastorFS - полноценная кластерная (Read-Write-Many) файловая система. Она поддерживает
|
||||
большую часть функций POSIX - иерархическую организацию, символические ссылки, жёсткие
|
||||
ссылки, быстрые переименования и так далее.
|
||||
|
||||
Метаданные VitastorFS хранятся в собственной реализации БД формата ключ-значения,
|
||||
основанной на Параллельном Оптимистичном Б-дереве поверх обычного блочного образа Vitastor.
|
||||
И записи каталогов, и иноды, как обычно в Vitastor, хранятся в простом человекочитаемом
|
||||
JSON-формате :-). Для инспекции содержимого БД можно использовать инструмент `vitastor-kv`.
|
||||
|
||||
Чтобы использовать VitastorFS:
|
||||
|
||||
1. Создайте пул для данных ФС или выберите существующий пустой пул
|
||||
2. Создайте блочный образ для метаданных ФС, желательно, в более быстром пуле (на SSD
|
||||
или по крайней мере на HDD, но без EC), но можно и в том же пуле, что данные
|
||||
(размер образа значения не имеет):
|
||||
`vitastor-cli create -s 10G -p fastpool testfs`
|
||||
3. Пометьте пул данных как ФС-пул: `vitastor-cli modify-pool --used-for-fs testfs data-pool`
|
||||
4. Либо примонтируйте ФС: `vitastor-nfs mount --fs testfs --pool data-pool /mnt/vita`
|
||||
5. Либо запустите сетевой NFS-сервер: `vitastor-nfs start --fs testfs --pool data-pool`
|
||||
|
||||
### Поддерживаемые функции POSIX
|
||||
|
||||
- Чтение актуальной версии данных сразу после записи
|
||||
- Последовательное и произвольное чтение и запись
|
||||
- Запись за пределами текущего размера файла
|
||||
- Иерархическая организация, мгновенное переименование файлов и каталогов
|
||||
- Изменение размера файла (truncate)
|
||||
- Права на файлы (chmod/chown)
|
||||
- Фиксация данных на диски (когда необходимо) (fsync)
|
||||
- Символические ссылки
|
||||
- Жёсткие ссылки
|
||||
- Специальные файлы (устройства, сокеты, каналы)
|
||||
- Отслеживание времён модификации (mtime), изменения атрибутов (ctime)
|
||||
- Ручное изменение времён модификации (mtime), последнего доступа (atime)
|
||||
- Корректная обработка изменений списка файлов во время листинга
|
||||
|
||||
### Ограничения
|
||||
|
||||
Отсутствующие на данный момент в VitastorFS функции POSIX:
|
||||
- Блокировки файлов не поддерживаются
|
||||
- Фактически занятое файлами место не подсчитывается и не возвращается вызовами
|
||||
stat(2), так что `du` всегда показывает сумму размеров файлов, а не фактически занятое место
|
||||
- Времена доступа (`atime`) не отслеживаются (как будто ФС смонтирована с `-o noatime`)
|
||||
- Времена модификации (`mtime`) отслеживаются асинхронно (как будто ФС смонтирована с `-o lazytime`)
|
||||
|
||||
Другие недостающие функции, которые нужно добавить в будущем:
|
||||
- Дефрагментация "общих инодов". На уровне реализации ФС файлы, меньшие, чем размер
|
||||
объекта пула (block_size умножить на число частей данных, если пул EC),
|
||||
упаковываются друг за другом в большие "общие" иноды/тома. Если такие файлы удалять
|
||||
или увеличивать, они перемещаются и оставляют за собой "мусор", вот тут-то и нужен
|
||||
дефрагментатор.
|
||||
- Переиспользование номеров инодов. В текущей реализации номера инодов всё время
|
||||
увеличиваются, так что в теории вы можете упереться в лимит, если насоздаёте
|
||||
и наудаляете больше, чем 2^48 файлов.
|
||||
- Очистка места в Б-дереве метаданных. Текущая реализация никогда не сливает и не
|
||||
удаляет блоки Б-дерева, так что в теории дерево может разростись и стать неоптимальным.
|
||||
Если вы столкнётесь с такой ситуацией сейчас, вы можете решить её с помощью
|
||||
команд `vitastor-kv dumpjson` и `loadjson` (т.е. пересоздав и загрузив обратно все метаданные ФС).
|
||||
- Инструмент проверки метаданных файловой системы. У VitastorFS нет журнала, так как
|
||||
журнал бы сильно замедлил реализацию, вместо него используются оптимистичные
|
||||
транзакции на основе CAS (сравнить-и-записать), и теоретически при нештатном
|
||||
завершении сервера ФС в БД также могут оставаться неконсистентные "мусорные"
|
||||
записи. ФС устроена так, что на работу они не влияют, но для порядка и их стоит
|
||||
уметь подчищать.
|
||||
|
||||
## Горизонтальное масштабирование
|
||||
|
||||
Клиент Linux NFS 3.0 не поддерживает встроенное масштабирование или отказоустойчивость.
|
||||
То есть, вы не можете задать несколько адресов серверов при монтировании ФС.
|
||||
|
||||
Однако вы можете использовать любые стандартные сетевые балансировщики нагрузки
|
||||
или схемы с отказоустойчивостью. Это точно безопасно при настройках `immediate_commit=all` и
|
||||
`client_enable_writeback=false`, так как с ними NFS-сервер Vitastor вообще не хранит
|
||||
в памяти ещё не зафиксированные на дисках данные; и вполне вероятно безопасно
|
||||
даже без `immediate_commit=all`, потому что NFS-клиент ядра Linux повторяет все
|
||||
незафиксированные запросы при потере соединения.
|
||||
|
||||
## Команды
|
||||
|
||||
### mount
|
||||
|
||||
`vitastor-nfs (--fs <NAME> | --block) mount [-o <OPT>] <MOUNTPOINT>`
|
||||
|
||||
Запустить локальный сервер и примонтировать ФС в директорию <MOUNTPOINT>.
|
||||
|
||||
Чтобы отмонтировать ФС, используйте обычную команду `umount <MOUNTPOINT>`.
|
||||
|
||||
Сервер автоматически останавливается при отмонтировании ФС.
|
||||
|
||||
- `-o|--options <OPT>` - Передать дополнительные опции монтирования NFS (пример: -o async).
|
||||
|
||||
### start
|
||||
|
||||
`vitastor-nfs (--fs <NAME> | --block) start`
|
||||
|
||||
Запустить сетевой NFS-сервер. Опции:
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|-----------------|-----------------------------------------------------------------------|
|
||||
| `--bind <IP>` | принимать соединения по адресу \<IP> (по умолчанию 0.0.0.0 - на всех) |
|
||||
| `--port <PORT>` | использовать порт \<PORT> для NFS-сервисов (по умолчанию 2049) |
|
||||
| `--portmap 0` | отключить сервис portmap/rpcbind на порту 111 (по умолчанию включён и требует root привилегий) |
|
||||
|
||||
## Общие опции
|
||||
|
||||
| <!-- --> | <!-- --> |
|
||||
|--------------------|---------------------------------------------------------|
|
||||
| `--fs <NAME>` | использовать VitastorFS с метаданными в образе \<NAME> |
|
||||
| `--block` | использовать псевдо-ФС для доступа к блочным образам |
|
||||
| `--pool <POOL>` | использовать пул \<POOL> для новых файлов (обязательно, если пул в кластере не один) |
|
||||
| `--subdir <DIR>` | экспортировать подкаталог \<DIR>, а не корень (только для псевдо-ФС) |
|
||||
| `--nfspath <PATH>` | установить путь NFS-экспорта в \<PATH> (по умолчанию /) |
|
||||
| `--pidfile <FILE>` | записать ID процесса в заданный файл |
|
||||
| `--logfile <FILE>` | записывать логи в заданный файл |
|
||||
| `--foreground 1` | не уходить в фон после запуска |
|
||||
|
|
|
@ -0,0 +1,13 @@
|
|||
digraph G {
|
||||
rankdir=LR;
|
||||
bgcolor=transparent;
|
||||
edge [color="#00A000"];
|
||||
node [shape=hexagon, fillcolor="#A0A000", fontcolor=white, fontname="sans-serif", fontsize=12, style=filled, penwidth=0];
|
||||
offline -> starting -> peering -> offline;
|
||||
stopping -> offline;
|
||||
starting -> incomplete -> offline;
|
||||
active -> repeering -> peering -> active -> stopping;
|
||||
offline [fillcolor="#A00000"];
|
||||
incomplete [fillcolor="#A00000"];
|
||||
active [fillcolor="#00A000"];
|
||||
}
|
|
@ -0,0 +1,114 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
|
||||
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<!-- Generated by graphviz version 2.43.0 (0)
|
||||
-->
|
||||
<!-- Title: G Pages: 1 -->
|
||||
<svg width="603pt" height="123pt"
|
||||
viewBox="0.00 0.00 602.66 122.55" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
|
||||
<g id="graph0" class="graph" transform="scale(1 1) rotate(0) translate(4 118.55)">
|
||||
<title>G</title>
|
||||
<!-- offline -->
|
||||
<g id="node1" class="node">
|
||||
<title>offline</title>
|
||||
<polygon fill="#a00000" stroke="black" stroke-width="0" points="75.52,-56 56.6,-74 18.75,-74 -0.17,-56 18.75,-38 56.6,-38 75.52,-56"/>
|
||||
<text text-anchor="middle" x="37.67" y="-52.9" font-family="sans-serif" font-size="12.00" fill="white">offline</text>
|
||||
</g>
|
||||
<!-- starting -->
|
||||
<g id="node2" class="node">
|
||||
<title>starting</title>
|
||||
<polygon fill="#a0a000" stroke="black" stroke-width="0" points="199.56,-79 177.49,-97 133.35,-97 111.28,-79 133.35,-61 177.49,-61 199.56,-79"/>
|
||||
<text text-anchor="middle" x="155.42" y="-75.9" font-family="sans-serif" font-size="12.00" fill="white">starting</text>
|
||||
</g>
|
||||
<!-- offline->starting -->
|
||||
<g id="edge1" class="edge">
|
||||
<title>offline->starting</title>
|
||||
<path fill="none" stroke="#00a000" d="M69.39,-62.1C81.66,-64.54 96.04,-67.4 109.45,-70.06"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="108.98,-73.54 119.47,-72.05 110.34,-66.67 108.98,-73.54"/>
|
||||
</g>
|
||||
<!-- peering -->
|
||||
<g id="node3" class="node">
|
||||
<title>peering</title>
|
||||
<polygon fill="#a0a000" stroke="black" stroke-width="0" points="335.57,-95 313.96,-113 270.74,-113 249.13,-95 270.74,-77 313.96,-77 335.57,-95"/>
|
||||
<text text-anchor="middle" x="292.35" y="-91.9" font-family="sans-serif" font-size="12.00" fill="white">peering</text>
|
||||
</g>
|
||||
<!-- starting->peering -->
|
||||
<g id="edge2" class="edge">
|
||||
<title>starting->peering</title>
|
||||
<path fill="none" stroke="#00a000" d="M194.36,-83.5C209.71,-85.32 227.6,-87.44 243.8,-89.36"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="243.82,-92.89 254.16,-90.59 244.64,-85.94 243.82,-92.89"/>
|
||||
</g>
|
||||
<!-- incomplete -->
|
||||
<g id="node5" class="node">
|
||||
<title>incomplete</title>
|
||||
<polygon fill="#a00000" stroke="black" stroke-width="0" points="349.09,-41 320.72,-59 263.99,-59 235.62,-41 263.99,-23 320.72,-23 349.09,-41"/>
|
||||
<text text-anchor="middle" x="292.35" y="-37.9" font-family="sans-serif" font-size="12.00" fill="white">incomplete</text>
|
||||
</g>
|
||||
<!-- starting->incomplete -->
|
||||
<g id="edge5" class="edge">
|
||||
<title>starting->incomplete</title>
|
||||
<path fill="none" stroke="#00a000" d="M188.74,-69.9C204.92,-65.34 224.85,-59.73 242.82,-54.67"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="243.9,-58 252.57,-51.92 242,-51.26 243.9,-58"/>
|
||||
</g>
|
||||
<!-- peering->offline -->
|
||||
<g id="edge3" class="edge">
|
||||
<title>peering->offline</title>
|
||||
<path fill="none" stroke="#00a000" d="M259.32,-103.69C222.67,-112.11 161.28,-121.52 111.35,-106 94.55,-100.78 78.2,-90.18 65.27,-80.08"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="67.26,-77.19 57.3,-73.58 62.84,-82.61 67.26,-77.19"/>
|
||||
</g>
|
||||
<!-- active -->
|
||||
<g id="node6" class="node">
|
||||
<title>active</title>
|
||||
<polygon fill="#00a000" stroke="black" stroke-width="0" points="456.34,-49 438.55,-67 402.97,-67 385.18,-49 402.97,-31 438.55,-31 456.34,-49"/>
|
||||
<text text-anchor="middle" x="420.76" y="-45.9" font-family="sans-serif" font-size="12.00" fill="white">active</text>
|
||||
</g>
|
||||
<!-- peering->active -->
|
||||
<g id="edge9" class="edge">
|
||||
<title>peering->active</title>
|
||||
<path fill="none" stroke="#00a000" d="M322.99,-84.22C341.47,-77.49 365.34,-68.8 384.75,-61.74"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="385.96,-65.03 394.16,-58.32 383.56,-58.45 385.96,-65.03"/>
|
||||
</g>
|
||||
<!-- stopping -->
|
||||
<g id="node4" class="node">
|
||||
<title>stopping</title>
|
||||
<polygon fill="#a0a000" stroke="black" stroke-width="0" points="591.65,-18 567.57,-36 519.39,-36 495.31,-18 519.39,0 567.57,0 591.65,-18"/>
|
||||
<text text-anchor="middle" x="543.48" y="-14.9" font-family="sans-serif" font-size="12.00" fill="white">stopping</text>
|
||||
</g>
|
||||
<!-- stopping->offline -->
|
||||
<g id="edge4" class="edge">
|
||||
<title>stopping->offline</title>
|
||||
<path fill="none" stroke="#00a000" d="M500.13,-14.3C440.78,-9.83 329.58,-4.07 235.49,-14 179.71,-19.89 116.5,-34.9 77.11,-45.29"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="76.14,-41.92 67.38,-47.89 77.94,-48.69 76.14,-41.92"/>
|
||||
</g>
|
||||
<!-- incomplete->offline -->
|
||||
<g id="edge6" class="edge">
|
||||
<title>incomplete->offline</title>
|
||||
<path fill="none" stroke="#00a000" d="M240.25,-44.03C194.33,-46.76 127.57,-50.72 83.64,-53.33"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="83.32,-49.84 73.54,-53.93 83.73,-56.83 83.32,-49.84"/>
|
||||
</g>
|
||||
<!-- active->stopping -->
|
||||
<g id="edge10" class="edge">
|
||||
<title>active->stopping</title>
|
||||
<path fill="none" stroke="#00a000" d="M449.46,-41.89C463.64,-38.25 481.26,-33.72 497.34,-29.59"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="498.29,-32.96 507.11,-27.08 496.55,-26.18 498.29,-32.96"/>
|
||||
</g>
|
||||
<!-- repeering -->
|
||||
<g id="node7" class="node">
|
||||
<title>repeering</title>
|
||||
<polygon fill="#a0a000" stroke="black" stroke-width="0" points="594.84,-83 569.16,-101 517.8,-101 492.12,-83 517.8,-65 569.16,-65 594.84,-83"/>
|
||||
<text text-anchor="middle" x="543.48" y="-79.9" font-family="sans-serif" font-size="12.00" fill="white">repeering</text>
|
||||
</g>
|
||||
<!-- active->repeering -->
|
||||
<g id="edge7" class="edge">
|
||||
<title>active->repeering</title>
|
||||
<path fill="none" stroke="#00a000" d="M448.85,-56.63C462.9,-60.59 480.44,-65.53 496.53,-70.06"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="495.74,-73.47 506.32,-72.82 497.64,-66.74 495.74,-73.47"/>
|
||||
</g>
|
||||
<!-- repeering->peering -->
|
||||
<g id="edge8" class="edge">
|
||||
<title>repeering->peering</title>
|
||||
<path fill="none" stroke="#00a000" d="M495.33,-85.27C451.99,-87.36 387.93,-90.44 343.63,-92.58"/>
|
||||
<polygon fill="#00a000" stroke="#00a000" points="343.2,-89.09 333.38,-93.07 343.54,-96.09 343.2,-89.09"/>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
After Width: | Height: | Size: 5.9 KiB |
|
@ -0,0 +1,408 @@
|
|||
const { select_murmur3 } = require('./murmur3.js');
|
||||
|
||||
const NO_OSD = 'Z';
|
||||
|
||||
class RuleCombinator
|
||||
{
|
||||
constructor(osd_tree, rules, max_combinations, ordered)
|
||||
{
|
||||
this.osd_tree = index_tree(Object.values(osd_tree).filter(o => o.id));
|
||||
this.rules = rules;
|
||||
this.max_combinations = max_combinations;
|
||||
this.ordered = ordered;
|
||||
}
|
||||
|
||||
random_combinations()
|
||||
{
|
||||
return random_custom_combinations(this.osd_tree, this.rules, this.max_combinations, this.ordered);
|
||||
}
|
||||
|
||||
check_combinations(pgs)
|
||||
{
|
||||
return check_custom_combinations(this.osd_tree, this.rules, pgs);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert alternative "level-index" format to rules
|
||||
// level_index = { [level: string]: string | string[] }
|
||||
// level_sequence = optional, levels from upper to lower, i.e. [ 'dc', 'host' ]
|
||||
// Example: level_index = { dc: "112233", host: "ABCDEF" }
|
||||
function parse_level_indexes(level_index, level_sequence)
|
||||
{
|
||||
const rules = [];
|
||||
const lvl_first = {};
|
||||
for (const level in level_index)
|
||||
{
|
||||
const idx = level_index[level];
|
||||
while (rules.length < idx.length)
|
||||
{
|
||||
rules.push([]);
|
||||
}
|
||||
const seen = {};
|
||||
for (let i = 0; i < idx.length; i++)
|
||||
{
|
||||
if (!seen[idx[i]])
|
||||
{
|
||||
const other = Object.values(seen);
|
||||
if (other.length)
|
||||
{
|
||||
rules[i].push([ level, '!=', other ]);
|
||||
}
|
||||
seen[idx[i]] = i+1;
|
||||
}
|
||||
else
|
||||
{
|
||||
rules[i].push([ level, '=', seen[idx[i]] ]);
|
||||
}
|
||||
}
|
||||
lvl_first[level] = seen;
|
||||
}
|
||||
if (level_sequence)
|
||||
{
|
||||
// Prune useless rules for the sake of prettiness
|
||||
// For simplicity, call "upper" level DC and "lower" level host
|
||||
const level_prio = Object.keys(level_sequence).reduce((a, c) => { a[level_sequence[c]] = c; return a; }, {});
|
||||
for (let upper_i = 0; upper_i < level_sequence.length-1; upper_i++)
|
||||
{
|
||||
const upper_level = level_sequence[upper_i];
|
||||
for (let i = 0; i < rules.length; i++)
|
||||
{
|
||||
const noteq = {};
|
||||
for (let k = 0; k < level_index[upper_level].length; k++)
|
||||
{
|
||||
// If upper_level[x] is different from upper_level[y]
|
||||
// then lower_level[x] is also different from lower_level[y]
|
||||
if (level_index[upper_level][k] != level_index[upper_level][i])
|
||||
{
|
||||
noteq[k+1] = true;
|
||||
}
|
||||
}
|
||||
for (let j = 0; j < rules[i].length; j++)
|
||||
{
|
||||
if (level_prio[rules[i][j][0]] != null && level_prio[rules[i][j][0]] > upper_i && rules[i][j][1] == '!=')
|
||||
{
|
||||
rules[i][j][2] = rules[i][j][2].filter(other_host => !noteq[other_host]);
|
||||
if (!rules[i][j][2].length)
|
||||
{
|
||||
rules[i].splice(j--, 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
|
||||
// Parse rules in DSL format
|
||||
// dsl := item | item ("\n" | ",") items
|
||||
// item := "any" | rules
|
||||
// rules := rule | rule rules
|
||||
// rule := level operator arg
|
||||
// level := /\w+/
|
||||
// operator := "!=" | "=" | ">" | "?="
|
||||
// arg := value | "(" values ")"
|
||||
// values := value | value "," values
|
||||
// value := item_ref | constant_id
|
||||
// item_ref := /\d+/
|
||||
// constant_id := /"([^"]+)"/
|
||||
//
|
||||
// Output: [ level, operator, value ][][]
|
||||
function parse_pg_dsl(text)
|
||||
{
|
||||
const tokens = [ ...text.matchAll(/\w+|!=|\?=|[>=\(\),\n]|"([^\"]+)"/g) ].map(t => [ t[0], t.index ]);
|
||||
let positions = [ [] ];
|
||||
let rules = positions[0];
|
||||
for (let i = 0; i < tokens.length; )
|
||||
{
|
||||
if (tokens[i][0] === '\n' || tokens[i][0] === ',')
|
||||
{
|
||||
rules = [];
|
||||
positions.push(rules);
|
||||
i++;
|
||||
}
|
||||
else if (!rules.length && tokens[i][0] === 'any' && (i == tokens.length-1 || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n'))
|
||||
{
|
||||
i++;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (!/^\w/.exec(tokens[i][0]))
|
||||
{
|
||||
throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (level name expected)');
|
||||
}
|
||||
if (i > tokens.length-3)
|
||||
{
|
||||
throw new Error('Unexpected EOF (operator and value expected)');
|
||||
}
|
||||
if (/^\w/.exec(tokens[i+1][0]) || tokens[i+1][0] === ',' || tokens[i+1][0] === '\n')
|
||||
{
|
||||
throw new Error('Unexpected '+tokens[i+1][0]+' at '+tokens[i+1][1]+' (operator expected)');
|
||||
}
|
||||
if (!/^[\w"(]/.exec(tokens[i+2][0])) // "
|
||||
{
|
||||
throw new Error('Unexpected '+tokens[i+2][0]+' at '+tokens[i+2][1]+' (id, round brace, number or node ID expected)');
|
||||
}
|
||||
let rule = [ tokens[i][0], tokens[i+1][0], tokens[i+2][0] ];
|
||||
i += 3;
|
||||
if (rule[2][0] == '"')
|
||||
{
|
||||
rule[2] = { id: rule[2].substr(1, rule[2].length-2) };
|
||||
}
|
||||
else if (rule[2] === '(')
|
||||
{
|
||||
rule[2] = [];
|
||||
while (true)
|
||||
{
|
||||
if (i > tokens.length-1)
|
||||
{
|
||||
throw new Error('Unexpected EOF (expected list and a closing round brace)');
|
||||
}
|
||||
if (tokens[i][0] === ',')
|
||||
{
|
||||
i++;
|
||||
}
|
||||
else if (tokens[i][0] === ')')
|
||||
{
|
||||
i++;
|
||||
break;
|
||||
}
|
||||
else if (tokens[i][0][0] === '"')
|
||||
{
|
||||
rule[2].push({ id: tokens[i][0].substr(1, tokens[i][0].length-2) });
|
||||
i++;
|
||||
}
|
||||
else if (/^\d+$/.exec(tokens[i][0]))
|
||||
{
|
||||
const n = 0|tokens[i][0];
|
||||
if (!n)
|
||||
{
|
||||
throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i][1]);
|
||||
}
|
||||
else if (n > positions.length)
|
||||
{
|
||||
throw new Error('Forward references are forbidden at '+tokens[i][1]);
|
||||
}
|
||||
rule[2].push(n);
|
||||
i++;
|
||||
}
|
||||
else if (!/^\w/.exec(tokens[i][0]))
|
||||
{
|
||||
throw new Error('Unexpected '+tokens[i][0]+' at '+tokens[i][1]+' (number or node ID expected)');
|
||||
}
|
||||
else
|
||||
{
|
||||
rule[2].push({ id: tokens[i][0] });
|
||||
i++;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (!/^\d+$/.exec(rule[2]))
|
||||
{
|
||||
rule[2] = { id: rule[2] };
|
||||
}
|
||||
else
|
||||
{
|
||||
rule[2] = 0|rule[2];
|
||||
if (!rule[2])
|
||||
{
|
||||
throw new Error('Level reference cannot be 0 (refs count from 1) at '+tokens[i-1][1]);
|
||||
}
|
||||
else if (rule[2] > positions.length)
|
||||
{
|
||||
throw new Error('Forward references are forbidden at '+tokens[i-1][1]);
|
||||
}
|
||||
}
|
||||
rules.push(rule);
|
||||
}
|
||||
}
|
||||
return positions;
|
||||
}
|
||||
|
||||
// osd_tree = index_tree() output
|
||||
// levels = { string: number }
|
||||
// rules = [ level, operator, value ][][]
|
||||
// level = string
|
||||
// operator = '=' | '!=' | '>' | '?='
|
||||
// value = number|number[] | { id: string|string[] }
|
||||
// examples:
|
||||
// 1) simple 3 replicas with failure_domain=host:
|
||||
// [ [], [ [ 'host', '!=', 1 ] ], [ [ 'host', '!=', [ 1, 2 ] ] ] ]
|
||||
// in DSL form: any, host!=1, host!=(1,2)
|
||||
// 2) EC 4+2 in 3 DC:
|
||||
// [ [], [ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
|
||||
// [ 'dc', '!=', 1 ], [ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
|
||||
// [ 'dc', '!=', [ 1, 3 ] ], [ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ] ]
|
||||
// in DSL form: any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5
|
||||
// 3) 1 replica in fixed DC + 2 in random DCs:
|
||||
// [ [ [ 'dc', '=', { id: 'meow' } ] ], [ [ 'dc', '!=', 1 ] ], [ [ 'dc', '!=', [ 1, 2 ] ] ] ]
|
||||
// in DSL form: dc=meow, dc!=1, dc!=(1,2)
|
||||
// 4) 2 replicas in each DC (almost the same as (2)):
|
||||
// DSL: any, dc=1 host!=1, dc!=1, dc=3 host!=3
|
||||
// Alternative simpler way to specify rules would be: [ DC: 112233 HOST: 123456 ]
|
||||
function random_custom_combinations(osd_tree, rules, count, ordered)
|
||||
{
|
||||
const r = {};
|
||||
const first = filter_tree_by_rules(osd_tree, rules[0], []);
|
||||
let max_size = 0;
|
||||
// All combinations for the first item (usually "any") to try to include each OSD at least once
|
||||
for (const f of first)
|
||||
{
|
||||
const selected = [ f ];
|
||||
for (let i = 1; i < rules.length; i++)
|
||||
{
|
||||
const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
|
||||
const idx = select_murmur3(filtered.length, i => 'p:'+f.id+':'+filtered[i].id);
|
||||
selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]);
|
||||
}
|
||||
const size = selected.filter(s => s.id !== null).length;
|
||||
max_size = max_size < size ? size : max_size;
|
||||
const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
|
||||
if (!ordered)
|
||||
pg.sort();
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
// Pseudo-random selection
|
||||
for (let n = 0; n < count; n++)
|
||||
{
|
||||
const selected = [];
|
||||
for (const item_rules of rules)
|
||||
{
|
||||
const filtered = selected.length ? filter_tree_by_rules(osd_tree, item_rules, selected) : first;
|
||||
const idx = select_murmur3(filtered.length, i => n+':'+filtered[i].id);
|
||||
selected.push(idx == null ? { levels: {}, id: null } : filtered[idx]);
|
||||
}
|
||||
const size = selected.filter(s => s.id !== null).length;
|
||||
max_size = max_size < size ? size : max_size;
|
||||
const pg = selected.map(s => s.id === null ? NO_OSD : s.id);
|
||||
if (!ordered)
|
||||
pg.sort();
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
// Exclude PGs with less successful selections than maximum
|
||||
for (const k in r)
|
||||
{
|
||||
if (r[k].filter(s => s !== NO_OSD).length < max_size)
|
||||
{
|
||||
delete r[k];
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
function filter_tree_by_rules(osd_tree, rules, selected)
|
||||
{
|
||||
let cur = osd_tree[''].children;
|
||||
for (const rule of rules)
|
||||
{
|
||||
const val = (rule[2] instanceof Array ? rule[2] : [ rule[2] ])
|
||||
.map(v => v instanceof Object ? v.id : selected[v-1].levels[rule[0]]);
|
||||
let preferred = [], other = [];
|
||||
for (let i = 0; i < cur.length; i++)
|
||||
{
|
||||
const item = cur[i];
|
||||
const level_id = item.levels[rule[0]];
|
||||
if (level_id)
|
||||
{
|
||||
if (rule[1] == '>' && val.filter(v => level_id <= v).length == 0 ||
|
||||
(rule[1] == '=' || rule[1] == '?=') && val.filter(v => level_id != v).length == 0 ||
|
||||
rule[1] == '!=' && val.filter(v => level_id == v).length == 0)
|
||||
{
|
||||
// Include
|
||||
preferred.push(item);
|
||||
}
|
||||
else if (rule[1] == '?=' && val.filter(v => level_id != v).length > 0)
|
||||
{
|
||||
// Non-preferred
|
||||
other.push(item);
|
||||
}
|
||||
}
|
||||
else if (item.children)
|
||||
{
|
||||
// Descend
|
||||
cur.splice(i+1, 0, ...item.children);
|
||||
}
|
||||
}
|
||||
cur = preferred.length ? preferred : other;
|
||||
}
|
||||
// Get leaf items
|
||||
for (let i = 0; i < cur.length; i++)
|
||||
{
|
||||
if (cur[i].children)
|
||||
{
|
||||
// Descend
|
||||
cur.splice(i, 1, ...cur[i].children);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
// Convert from
|
||||
// node_list = { id: string|number, level: string, size?: number, parent?: string|number }[]
|
||||
// to
|
||||
// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node_id[], levels: { [level]: id, ... } } }
|
||||
function index_tree(node_list)
|
||||
{
|
||||
const tree = { '': { children: [], levels: {} } };
|
||||
for (const node of node_list)
|
||||
{
|
||||
tree[node.id] = { ...node, levels: {} };
|
||||
delete tree[node.id].children;
|
||||
}
|
||||
for (const node of node_list)
|
||||
{
|
||||
const parent_id = node.parent && tree[node.parent] ? node.parent : '';
|
||||
tree[parent_id].children = tree[parent_id].children || [];
|
||||
tree[parent_id].children.push(tree[node.id]);
|
||||
}
|
||||
const cur = tree[''].children;
|
||||
for (let i = 0; i < cur.length; i++)
|
||||
{
|
||||
cur[i].levels[cur[i].level] = cur[i].id;
|
||||
if (cur[i].children)
|
||||
{
|
||||
for (const child of cur[i].children)
|
||||
{
|
||||
child.levels = { ...cur[i].levels, ...child.levels };
|
||||
}
|
||||
cur.splice(i, 1, ...cur[i].children);
|
||||
i--;
|
||||
}
|
||||
}
|
||||
return tree;
|
||||
}
|
||||
|
||||
// selection = id[]
|
||||
// osd_tree = index_tree output
|
||||
// rules = parse_pg_dsl output
|
||||
function check_custom_combinations(osd_tree, rules, pgs)
|
||||
{
|
||||
const res = [];
|
||||
skip_pg: for (const pg of pgs)
|
||||
{
|
||||
let selected = pg.map(id => osd_tree[id] || null);
|
||||
for (let i = 0; i < rules.length; i++)
|
||||
{
|
||||
const filtered = filter_tree_by_rules(osd_tree, rules[i], selected);
|
||||
if (selected[i] === null && filtered.length ||
|
||||
!filtered.filter(ok => selected[i].id === ok.id).length)
|
||||
{
|
||||
continue skip_pg;
|
||||
}
|
||||
}
|
||||
res.push(pg);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
RuleCombinator,
|
||||
NO_OSD,
|
||||
|
||||
index_tree,
|
||||
parse_level_indexes,
|
||||
parse_pg_dsl,
|
||||
random_custom_combinations,
|
||||
check_custom_combinations,
|
||||
};
|
|
@ -50,15 +50,15 @@ async function lp_solve(text)
|
|||
return { score, vars };
|
||||
}
|
||||
|
||||
async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
|
||||
// osd_weights = { [id]: weight }
|
||||
async function optimize_initial({ osd_weights, combinator, pg_count, pg_size = 3, pg_minsize = 2, parity_space = 1, ordered = false })
|
||||
{
|
||||
if (!pg_count || !osd_tree)
|
||||
if (!pg_count || !osd_weights)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
const all_weights = Object.assign({}, ...Object.values(osd_tree));
|
||||
const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
|
||||
const total_weight = Object.values(osd_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const all_pgs = Object.values(make_cyclic(combinator.random_combinations(), parity_space));
|
||||
const pg_per_osd = {};
|
||||
for (const pg of all_pgs)
|
||||
{
|
||||
|
@ -69,15 +69,15 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
|
|||
pg_per_osd[osd].push((i >= pg_minsize ? parity_space+'*' : '')+"pg_"+pg.join("_"));
|
||||
}
|
||||
}
|
||||
const pg_effsize = Math.min(pg_minsize, Object.keys(osd_tree).length)
|
||||
+ Math.max(0, Math.min(pg_size, Object.keys(osd_tree).length) - pg_minsize) * parity_space;
|
||||
let pg_effsize = all_pgs.reduce((a, c) => Math.max(a, c.filter(e => e != NO_OSD).length), 0);
|
||||
pg_effsize = Math.min(pg_minsize, pg_effsize) + Math.max(0, Math.min(pg_size, pg_effsize) - pg_minsize) * parity_space;
|
||||
let lp = '';
|
||||
lp += "max: "+all_pgs.map(pg => 'pg_'+pg.join('_')).join(' + ')+";\n";
|
||||
for (const osd in pg_per_osd)
|
||||
{
|
||||
if (osd !== NO_OSD)
|
||||
{
|
||||
let osd_pg_count = all_weights[osd]/total_weight*pg_effsize*pg_count;
|
||||
let osd_pg_count = osd_weights[osd]/total_weight*pg_effsize*pg_count;
|
||||
lp += pg_per_osd[osd].join(' + ')+' <= '+osd_pg_count+';\n';
|
||||
}
|
||||
}
|
||||
|
@ -93,7 +93,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
|
|||
throw new Error('Problem is infeasible or unbounded - is it a bug?');
|
||||
}
|
||||
const int_pgs = make_int_pgs(lp_result.vars, pg_count, ordered);
|
||||
const eff = pg_list_space_efficiency(int_pgs, all_weights, pg_minsize, parity_space);
|
||||
const eff = pg_list_space_efficiency(int_pgs, osd_weights, pg_minsize, parity_space);
|
||||
const res = {
|
||||
score: lp_result.score,
|
||||
weights: lp_result.vars,
|
||||
|
@ -104,6 +104,22 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
|
|||
return res;
|
||||
}
|
||||
|
||||
function make_cyclic(pgs, parity_space)
|
||||
{
|
||||
if (parity_space > 1)
|
||||
{
|
||||
for (const pg in pgs)
|
||||
{
|
||||
for (let i = 1; i < pg.size; i++)
|
||||
{
|
||||
const cyclic = [ ...pg.slice(i), ...pg.slice(0, i) ];
|
||||
pgs['pg_'+cyclic.join('_')] = cyclic;
|
||||
}
|
||||
}
|
||||
}
|
||||
return pgs;
|
||||
}
|
||||
|
||||
function shuffle(array)
|
||||
{
|
||||
for (let i = array.length - 1, j, x; i > 0; i--)
|
||||
|
@ -216,47 +232,17 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
|
|||
return move_weights;
|
||||
}
|
||||
|
||||
function add_valid_previous(osd_tree, prev_weights, all_pgs)
|
||||
{
|
||||
// Add previous combinations that are still valid
|
||||
const hosts = Object.keys(osd_tree).sort();
|
||||
const host_per_osd = {};
|
||||
for (const host in osd_tree)
|
||||
{
|
||||
for (const osd in osd_tree[host])
|
||||
{
|
||||
host_per_osd[osd] = host;
|
||||
}
|
||||
}
|
||||
skip_pg: for (const pg_name in prev_weights)
|
||||
{
|
||||
const seen_hosts = {};
|
||||
const pg = pg_name.substr(3).split(/_/);
|
||||
for (const osd of pg)
|
||||
{
|
||||
if (!host_per_osd[osd] || seen_hosts[host_per_osd[osd]])
|
||||
{
|
||||
continue skip_pg;
|
||||
}
|
||||
seen_hosts[host_per_osd[osd]] = true;
|
||||
}
|
||||
if (!all_pgs[pg_name])
|
||||
{
|
||||
all_pgs[pg_name] = pg;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Try to minimize data movement
|
||||
async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
|
||||
async function optimize_change({ prev_pgs: prev_int_pgs, osd_weights, combinator, pg_size = 3, pg_minsize = 2, parity_space = 1, ordered = false })
|
||||
{
|
||||
if (!osd_tree)
|
||||
if (!osd_weights)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
// FIXME: use parity_chunks with parity_space instead of pg_minsize
|
||||
const pg_effsize = Math.min(pg_minsize, Object.keys(osd_tree).length)
|
||||
+ Math.max(0, Math.min(pg_size, Object.keys(osd_tree).length) - pg_minsize) * parity_space;
|
||||
let all_pgs = make_cyclic(combinator.random_combinations(), parity_space);
|
||||
let pg_effsize = Object.values(all_pgs).reduce((a, c) => Math.max(a, c.filter(e => e != NO_OSD).length), 0);
|
||||
pg_effsize = Math.min(pg_minsize, pg_effsize) + Math.max(0, Math.min(pg_size, pg_effsize) - pg_minsize) * parity_space;
|
||||
const pg_count = prev_int_pgs.length;
|
||||
const prev_weights = {};
|
||||
const prev_pg_per_osd = {};
|
||||
|
@ -273,10 +259,13 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
}
|
||||
const old_pg_size = prev_int_pgs[0].length;
|
||||
// Get all combinations
|
||||
let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
|
||||
if (old_pg_size == pg_size)
|
||||
{
|
||||
add_valid_previous(osd_tree, prev_weights, all_pgs);
|
||||
const still_valid = combinator.check_combinations(Object.keys(prev_weights).map(pg_name => pg_name.substr(3).split('_')));
|
||||
for (const pg of still_valid)
|
||||
{
|
||||
all_pgs['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
}
|
||||
all_pgs = Object.values(all_pgs);
|
||||
const pg_per_osd = {};
|
||||
|
@ -295,8 +284,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
// Calculate total weight - old PG weights
|
||||
const all_pg_names = all_pgs.map(pg => 'pg_'+pg.join('_'));
|
||||
const all_pgs_hash = all_pg_names.reduce((a, c) => { a[c] = true; return a; }, {});
|
||||
const all_weights = Object.assign({}, ...Object.values(osd_tree));
|
||||
const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const total_weight = Object.values(osd_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
// Generate the LP problem
|
||||
let lp = '';
|
||||
lp += 'max: '+all_pg_names.map(pg_name => (
|
||||
|
@ -311,7 +299,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
)).join(' + ');
|
||||
const rm_osd_pg_count = (prev_pg_per_osd[osd]||[])
|
||||
.reduce((a, [ old_pg_name, space ]) => (a + (all_pgs_hash[old_pg_name] ? space : 0)), 0);
|
||||
const osd_pg_count = all_weights[osd]*pg_effsize/total_weight*pg_count - rm_osd_pg_count;
|
||||
const osd_pg_count = osd_weights[osd]*pg_effsize/total_weight*pg_count - rm_osd_pg_count;
|
||||
lp += osd_sum + ' <= ' + osd_pg_count + ';\n';
|
||||
}
|
||||
}
|
||||
|
@ -421,7 +409,7 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
|
|||
int_pgs: new_pgs,
|
||||
differs,
|
||||
osd_differs,
|
||||
space: pg_effsize * pg_list_space_efficiency(new_pgs, all_weights, pg_minsize, parity_space),
|
||||
space: pg_effsize * pg_list_space_efficiency(new_pgs, osd_weights, pg_minsize, parity_space),
|
||||
total_space: total_weight,
|
||||
};
|
||||
}
|
||||
|
@ -502,198 +490,6 @@ function put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, keygen)
|
|||
}
|
||||
}
|
||||
|
||||
// Convert multi-level osd_tree = { level: number|string, id?: string, size?: number, children?: osd_tree }[]
|
||||
// levels = { string: number }
|
||||
// to a two-level osd_tree suitable for all_combinations()
|
||||
function flatten_tree(osd_tree, levels, failure_domain_level, osd_level, domains = {}, i = { i: 1 })
|
||||
{
|
||||
osd_level = levels[osd_level] || osd_level;
|
||||
failure_domain_level = levels[failure_domain_level] || failure_domain_level;
|
||||
for (const node of osd_tree)
|
||||
{
|
||||
if ((levels[node.level] || node.level) < failure_domain_level)
|
||||
{
|
||||
flatten_tree(node.children||[], levels, failure_domain_level, osd_level, domains, i);
|
||||
}
|
||||
else
|
||||
{
|
||||
domains['dom'+(i.i++)] = extract_osds([ node ], levels, osd_level);
|
||||
}
|
||||
}
|
||||
return domains;
|
||||
}
|
||||
|
||||
function extract_osds(osd_tree, levels, osd_level, osds = {})
|
||||
{
|
||||
for (const node of osd_tree)
|
||||
{
|
||||
if ((levels[node.level] || node.level) >= osd_level)
|
||||
{
|
||||
osds[node.id] = node.size;
|
||||
}
|
||||
else
|
||||
{
|
||||
extract_osds(node.children||[], levels, osd_level, osds);
|
||||
}
|
||||
}
|
||||
return osds;
|
||||
}
|
||||
|
||||
// ordered = don't treat (x,y) and (y,x) as equal
|
||||
function random_combinations(osd_tree, pg_size, count, ordered)
|
||||
{
|
||||
let seed = 0x5f020e43;
|
||||
let rng = () =>
|
||||
{
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
return seed + 2147483648;
|
||||
};
|
||||
const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
|
||||
const hosts = Object.keys(osd_tree).sort().filter(h => osds[h].length > 0);
|
||||
const r = {};
|
||||
// Generate random combinations including each OSD at least once
|
||||
for (let h = 0; h < hosts.length; h++)
|
||||
{
|
||||
for (let o = 0; o < osds[hosts[h]].length; o++)
|
||||
{
|
||||
const pg = [ osds[hosts[h]][o] ];
|
||||
const cur_hosts = [ ...hosts ];
|
||||
cur_hosts.splice(h, 1);
|
||||
for (let i = 1; i < pg_size && i < hosts.length; i++)
|
||||
{
|
||||
const next_host = rng() % cur_hosts.length;
|
||||
const next_osd = rng() % osds[cur_hosts[next_host]].length;
|
||||
pg.push(osds[cur_hosts[next_host]][next_osd]);
|
||||
cur_hosts.splice(next_host, 1);
|
||||
}
|
||||
const cyclic_pgs = [ pg ];
|
||||
if (ordered)
|
||||
{
|
||||
for (let i = 1; i < pg.size; i++)
|
||||
{
|
||||
cyclic_pgs.push([ ...pg.slice(i), ...pg.slice(0, i) ]);
|
||||
}
|
||||
}
|
||||
for (const pg of cyclic_pgs)
|
||||
{
|
||||
while (pg.length < pg_size)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Generate purely random combinations
|
||||
while (count > 0)
|
||||
{
|
||||
let host_idx = [];
|
||||
const cur_hosts = [ ...hosts.map((h, i) => i) ];
|
||||
const max_hosts = pg_size < hosts.length ? pg_size : hosts.length;
|
||||
if (ordered)
|
||||
{
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
const r = rng() % cur_hosts.length;
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(r, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
const r = rng() % (cur_hosts.length - (max_hosts - i - 1));
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(0, r+1);
|
||||
}
|
||||
}
|
||||
let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
|
||||
while (pg.length < pg_size)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
count--;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// Super-stupid algorithm. Given the current OSD tree, generate all possible OSD combinations
|
||||
// osd_tree = { failure_domain1: { osd1: size1, ... }, ... }
|
||||
// ordered = return combinations without duplicates having different order
|
||||
function all_combinations(osd_tree, pg_size, ordered, count)
|
||||
{
|
||||
const hosts = Object.keys(osd_tree).sort();
|
||||
const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
|
||||
while (hosts.length < pg_size)
|
||||
{
|
||||
osds[NO_OSD] = [ NO_OSD ];
|
||||
hosts.push(NO_OSD);
|
||||
}
|
||||
let host_idx = [];
|
||||
let osd_idx = [];
|
||||
for (let i = 0; i < pg_size; i++)
|
||||
{
|
||||
host_idx.push(i);
|
||||
osd_idx.push(0);
|
||||
}
|
||||
const r = [];
|
||||
while (!count || count < 0 || r.length < count)
|
||||
{
|
||||
r.push(host_idx.map((hi, i) => osds[hosts[hi]][osd_idx[i]]));
|
||||
let inc = pg_size-1;
|
||||
while (inc >= 0)
|
||||
{
|
||||
osd_idx[inc]++;
|
||||
if (osd_idx[inc] >= osds[hosts[host_idx[inc]]].length)
|
||||
{
|
||||
osd_idx[inc] = 0;
|
||||
inc--;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (inc < 0)
|
||||
{
|
||||
// no osds left in the current host combination, select the next one
|
||||
inc = pg_size-1;
|
||||
same_again: while (inc >= 0)
|
||||
{
|
||||
host_idx[inc]++;
|
||||
for (let prev_host = 0; prev_host < inc; prev_host++)
|
||||
{
|
||||
if (host_idx[prev_host] == host_idx[inc])
|
||||
{
|
||||
continue same_again;
|
||||
}
|
||||
}
|
||||
if (host_idx[inc] < (ordered ? hosts.length-(pg_size-1-inc) : hosts.length))
|
||||
{
|
||||
while ((++inc) < pg_size)
|
||||
{
|
||||
host_idx[inc] = (ordered ? host_idx[inc-1]+1 : 0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
inc--;
|
||||
}
|
||||
}
|
||||
if (inc < 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
function pg_weights_space_efficiency(weights, pg_count, osd_sizes)
|
||||
{
|
||||
const per_osd = {};
|
||||
|
@ -752,11 +548,8 @@ module.exports = {
|
|||
pg_weights_space_efficiency,
|
||||
pg_list_space_efficiency,
|
||||
pg_per_osd_space_efficiency,
|
||||
flatten_tree,
|
||||
|
||||
lp_solve,
|
||||
make_int_pgs,
|
||||
align_pgs,
|
||||
random_combinations,
|
||||
all_combinations,
|
||||
};
|
||||
|
|
149
mon/mon.js
149
mon/mon.js
|
@ -6,6 +6,8 @@ const http = require('http');
|
|||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
const WebSocket = require('ws');
|
||||
const { RuleCombinator, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
|
||||
const { SimpleCombinator } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
const stableStringify = require('./stable-stringify.js');
|
||||
const PGUtil = require('./PGUtil.js');
|
||||
|
@ -37,7 +39,7 @@ const etcd_allow = new RegExp('^'+[
|
|||
'pg/history/[1-9]\\d*/[1-9]\\d*',
|
||||
'pool/stats/[1-9]\\d*',
|
||||
'history/last_clean_pgs',
|
||||
'inode/stats/[1-9]\\d*/[1-9]\\d*',
|
||||
'inode/stats/[1-9]\\d*/\\d+',
|
||||
'pool/stats/[1-9]\\d*',
|
||||
'stats',
|
||||
'index/image/.*',
|
||||
|
@ -63,6 +65,7 @@ const etcd_tree = {
|
|||
mon_stats_timeout: 1000, // ms. min: 100
|
||||
osd_out_time: 600, // seconds. min: 0
|
||||
placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
|
||||
force_new_pg_combinator: false,
|
||||
// client and osd
|
||||
tcp_header_buffer_size: 65536,
|
||||
use_sync_send_recv: false,
|
||||
|
@ -86,13 +89,14 @@ const etcd_tree = {
|
|||
client_max_buffered_bytes: 33554432,
|
||||
client_max_buffered_ops: 1024,
|
||||
client_max_writeback_iodepth: 256,
|
||||
client_retry_interval: 50, // ms. min: 10
|
||||
client_eio_retry_interval: 1000, // ms
|
||||
// client and osd - configurable online
|
||||
log_level: 0,
|
||||
peer_connect_interval: 5, // seconds. min: 1
|
||||
peer_connect_timeout: 5, // seconds. min: 1
|
||||
osd_idle_timeout: 5, // seconds. min: 1
|
||||
osd_ping_timeout: 5, // seconds. min: 1
|
||||
up_wait_retry_interval: 50, // ms. min: 10
|
||||
max_etcd_attempts: 5,
|
||||
etcd_quick_timeout: 1000, // ms
|
||||
etcd_slow_timeout: 5000, // ms
|
||||
|
@ -184,7 +188,10 @@ const etcd_tree = {
|
|||
// number of parity chunks, required for EC
|
||||
parity_chunks?: 1,
|
||||
pg_count: 100,
|
||||
failure_domain: 'host',
|
||||
// default is failure_domain=host
|
||||
failure_domain?: 'host',
|
||||
level_rules?: { host: '123' },
|
||||
pg_rules?: 'any, dc=1 host!=1, dc=1 host!=(1,2)',
|
||||
max_osd_combinations: 10000,
|
||||
// block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
|
||||
block_size: 131072,
|
||||
|
@ -929,7 +936,6 @@ class Mon
|
|||
// Parent's level must be less than child's; OSDs must be leaves
|
||||
const parent = parent_level && parent_level < node_level ? node_cfg.parent : '';
|
||||
tree[parent].children.push(tree[node_id]);
|
||||
delete node_cfg.parent;
|
||||
}
|
||||
return { up_osds, levels, osd_tree: tree };
|
||||
}
|
||||
|
@ -1095,7 +1101,6 @@ class Mon
|
|||
pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
|
||||
pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
|
||||
pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
|
||||
pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
|
||||
pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
|
||||
if (!/^[1-9]\d*$/.exec(''+pool_id))
|
||||
{
|
||||
|
@ -1175,10 +1180,32 @@ class Mon
|
|||
console.log('Pool '+pool_id+' has invalid primary_affinity_tags (must be a string or array of strings)');
|
||||
return false;
|
||||
}
|
||||
if (!this.get_pg_rules(pool_id, pool_cfg, true))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
filter_osds_by_tags(orig_tree, flat_tree, tags)
|
||||
filter_osds_by_root_node(pool_tree, root_node)
|
||||
{
|
||||
if (!root_node)
|
||||
{
|
||||
return;
|
||||
}
|
||||
pool_tree = pool_tree[pool_cfg.root_node];
|
||||
const cur = [ ...(pool_tree||{}).children||[] ];
|
||||
for (let i = 0; i < cur.length; i++)
|
||||
{
|
||||
if (cur.children)
|
||||
{
|
||||
cur.splice(i+1, 1, ...cur.children);
|
||||
}
|
||||
}
|
||||
return cur;
|
||||
}
|
||||
|
||||
filter_osds_by_tags(orig_tree, tags)
|
||||
{
|
||||
if (!tags)
|
||||
{
|
||||
|
@ -1186,30 +1213,22 @@ class Mon
|
|||
}
|
||||
for (const tag of (tags instanceof Array ? tags : [ tags ]))
|
||||
{
|
||||
for (const host in flat_tree)
|
||||
for (const osd in orig_tree)
|
||||
{
|
||||
let found = 0;
|
||||
for (const osd in flat_tree[host])
|
||||
if (orig_tree[osd].level === 'osd' &&
|
||||
(!orig_tree[osd].tags || !orig_tree[osd].tags[tag]))
|
||||
{
|
||||
if (!orig_tree[osd].tags || !orig_tree[osd].tags[tag])
|
||||
delete flat_tree[host][osd];
|
||||
else
|
||||
found++;
|
||||
}
|
||||
if (!found)
|
||||
{
|
||||
delete flat_tree[host];
|
||||
delete orig_tree[osd];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
filter_osds_by_block_layout(flat_tree, block_size, bitmap_granularity, immediate_commit)
|
||||
filter_osds_by_block_layout(orig_tree, block_size, bitmap_granularity, immediate_commit)
|
||||
{
|
||||
for (const host in flat_tree)
|
||||
for (const osd in orig_tree)
|
||||
{
|
||||
let found = 0;
|
||||
for (const osd in flat_tree[host])
|
||||
if (orig_tree[osd].level === 'osd')
|
||||
{
|
||||
const osd_stat = this.state.osd.stats[osd];
|
||||
if (osd_stat && (osd_stat.bs_block_size && osd_stat.bs_block_size != block_size ||
|
||||
|
@ -1217,16 +1236,8 @@ class Mon
|
|||
osd_stat.immediate_commit == 'small' && immediate_commit == 'all' ||
|
||||
osd_stat.immediate_commit == 'none' && immediate_commit != 'none'))
|
||||
{
|
||||
delete flat_tree[host][osd];
|
||||
delete orig_tree[host][osd];
|
||||
}
|
||||
else
|
||||
{
|
||||
found++;
|
||||
}
|
||||
}
|
||||
if (!found)
|
||||
{
|
||||
delete flat_tree[host];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -1236,12 +1247,60 @@ class Mon
|
|||
let aff_osds = up_osds;
|
||||
if (pool_cfg.primary_affinity_tags)
|
||||
{
|
||||
aff_osds = { ...up_osds };
|
||||
this.filter_osds_by_tags(osd_tree, { x: aff_osds }, pool_cfg.primary_affinity_tags);
|
||||
aff_osds = Object.keys(up_osds).reduce((a, c) => { a[c] = osd_tree[c]; return a; }, {});
|
||||
this.filter_osds_by_tags(aff_osds, pool_cfg.primary_affinity_tags);
|
||||
for (const osd in aff_osds)
|
||||
{
|
||||
aff_osds[osd] = true;
|
||||
}
|
||||
}
|
||||
return aff_osds;
|
||||
}
|
||||
|
||||
get_pg_rules(pool_id, pool_cfg, warn)
|
||||
{
|
||||
if (pool_cfg.level_rules instanceof Object)
|
||||
{
|
||||
const levels = this.config.placement_levels||{};
|
||||
levels.host = levels.host || 100;
|
||||
levels.osd = levels.osd || 101;
|
||||
for (const k in pool_cfg.level_rules)
|
||||
{
|
||||
if (!levels[k] || typeof pool_cfg.level_rules[k] !== 'string' &&
|
||||
(!pool_cfg.level_rules[k] instanceof Array || pool_cfg.level_rules[k].filter(s => typeof s !== 'string').length > 0))
|
||||
{
|
||||
if (warn)
|
||||
console.log('Pool '+pool_id+' configuration is invalid: level_rules should be { [level]: string or array of strings }');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
return parse_level_indexes(pool_cfg.level_rules);
|
||||
}
|
||||
else if (typeof pool_cfg.pg_rules === 'string')
|
||||
{
|
||||
try
|
||||
{
|
||||
return parse_pg_dsl(pool_cfg.pg_rules);
|
||||
}
|
||||
catch (e)
|
||||
{
|
||||
if (warn)
|
||||
console.log('Pool '+pool_id+' configuration is invalid: invalid pg_rules: '+e.message);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
let rules = [ [] ];
|
||||
let prev = [ 1 ];
|
||||
for (let i = 1; i < pool_cfg.pg_size; i++)
|
||||
{
|
||||
rules.push([ [ pool_cfg.failure_domain||'host', '!=', prev ] ]);
|
||||
prev = [ ...prev, i+1 ];
|
||||
}
|
||||
return rules;
|
||||
}
|
||||
}
|
||||
|
||||
async generate_pool_pgs(pool_id, osd_tree, levels)
|
||||
{
|
||||
const pool_cfg = this.state.config.pools[pool_id];
|
||||
|
@ -1249,10 +1308,9 @@ class Mon
|
|||
{
|
||||
return null;
|
||||
}
|
||||
let pool_tree = osd_tree[pool_cfg.root_node || ''];
|
||||
pool_tree = pool_tree ? pool_tree.children : [];
|
||||
pool_tree = LPOptimizer.flatten_tree(pool_tree, levels, pool_cfg.failure_domain, 'osd');
|
||||
this.filter_osds_by_tags(osd_tree, pool_tree, pool_cfg.osd_tags);
|
||||
let pool_tree = osd_tree;
|
||||
this.filter_osds_by_root_node(pool_tree, pool_cfg.root_node);
|
||||
this.filter_osds_by_tags(pool_tree, pool_cfg.osd_tags);
|
||||
this.filter_osds_by_block_layout(
|
||||
pool_tree,
|
||||
pool_cfg.block_size || this.config.block_size || 131072,
|
||||
|
@ -1275,11 +1333,15 @@ class Mon
|
|||
}
|
||||
const old_pg_count = prev_pgs.length;
|
||||
const optimize_cfg = {
|
||||
osd_tree: pool_tree,
|
||||
osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}),
|
||||
combinator: 1 || this.config.force_new_pg_combinator || !pool_cfg.level_rules && !pool_cfg.pg_rules
|
||||
// new algorithm:
|
||||
? new RuleCombinator(osd_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations)
|
||||
// old algorithm:
|
||||
: new SimpleCombinator(flatten_tree(osd_tree, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
|
||||
pg_count: pool_cfg.pg_count,
|
||||
pg_size: pool_cfg.pg_size,
|
||||
pg_minsize: pool_cfg.pg_minsize,
|
||||
max_combinations: pool_cfg.max_osd_combinations,
|
||||
ordered: pool_cfg.scheme != 'replicated',
|
||||
};
|
||||
let optimize_result;
|
||||
|
@ -1348,8 +1410,8 @@ class Mon
|
|||
// Something has changed
|
||||
console.log('Pool configuration or OSD tree changed, re-optimizing');
|
||||
// First re-optimize PGs, but don't look at history yet
|
||||
const optimize_results = await Promise.all(Object.keys(this.state.config.pools)
|
||||
.map(pool_id => this.generate_pool_pgs(pool_id, osd_tree, levels)));
|
||||
const optimize_results = (await Promise.all(Object.keys(this.state.config.pools)
|
||||
.map(pool_id => this.generate_pool_pgs(pool_id, osd_tree, levels)))).filter(r => r);
|
||||
// Then apply the modification in the form of an optimistic transaction,
|
||||
// each time considering new pg/history modifications (OSDs modify it during rebalance)
|
||||
while (!await this.apply_pool_pgs(optimize_results, up_osds, osd_tree, tree_hash))
|
||||
|
@ -1435,6 +1497,10 @@ class Mon
|
|||
|
||||
async apply_pool_pgs(results, up_osds, osd_tree, tree_hash)
|
||||
{
|
||||
if (!results.length)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
for (const pool_id in (this.state.config.pgs||{}).items||{})
|
||||
{
|
||||
// We should stop all PGs when deleting a pool or changing its PG count
|
||||
|
@ -1736,8 +1802,11 @@ class Mon
|
|||
for (const inode_num in this.state.osd.space[osd_num][pool_id])
|
||||
{
|
||||
const u = BigInt(this.state.osd.space[osd_num][pool_id][inode_num]||0);
|
||||
if (inode_num)
|
||||
{
|
||||
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
||||
inode_stats[pool_id][inode_num].raw_used += u;
|
||||
}
|
||||
this.state.pool.stats[pool_id].used_raw_tb += u;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -0,0 +1,38 @@
|
|||
function select_murmur3(count, cb)
|
||||
{
|
||||
if (!count)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
else
|
||||
{
|
||||
let i = 0, maxh = -1;
|
||||
for (let j = 0; j < count; j++)
|
||||
{
|
||||
const h = murmur3(cb(j));
|
||||
if (h > maxh)
|
||||
{
|
||||
i = j;
|
||||
maxh = h;
|
||||
}
|
||||
}
|
||||
return i;
|
||||
}
|
||||
}
|
||||
|
||||
function murmur3(s)
|
||||
{
|
||||
let hash = 0x12345678;
|
||||
for (let i = 0; i < s.length; i++)
|
||||
{
|
||||
hash ^= s.charCodeAt(i);
|
||||
hash = (hash*0x5bd1e995) & 0xFFFFFFFF;
|
||||
hash ^= (hash >> 15);
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
murmur3,
|
||||
select_murmur3,
|
||||
};
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"name": "vitastor-mon",
|
||||
"version": "1.4.7",
|
||||
"version": "1.5.0",
|
||||
"description": "Vitastor SDS monitor service",
|
||||
"main": "mon-main.js",
|
||||
"scripts": {
|
||||
|
|
|
@ -0,0 +1,247 @@
|
|||
const NO_OSD = 'Z';
|
||||
|
||||
class SimpleCombinator
|
||||
{
|
||||
constructor(flat_tree, pg_size, max_combinations, ordered)
|
||||
{
|
||||
this.osd_tree = flat_tree;
|
||||
this.pg_size = pg_size;
|
||||
this.max_combinations = max_combinations;
|
||||
this.ordered = ordered;
|
||||
}
|
||||
|
||||
random_combinations()
|
||||
{
|
||||
return random_combinations(this.osd_tree, this.pg_size, this.max_combinations, this.ordered);
|
||||
}
|
||||
|
||||
check_combinations(pgs)
|
||||
{
|
||||
return check_combinations(this.osd_tree, pgs);
|
||||
}
|
||||
}
|
||||
|
||||
// Convert multi-level osd_tree = { level: number|string, id?: string, size?: number, children?: osd_tree }[]
|
||||
// levels = { string: number }
|
||||
// to a two-level osd_tree suitable for all_combinations()
|
||||
function flatten_tree(osd_tree, levels, failure_domain_level, osd_level, domains = {}, i = { i: 1 })
|
||||
{
|
||||
osd_level = levels[osd_level] || osd_level;
|
||||
failure_domain_level = levels[failure_domain_level] || failure_domain_level;
|
||||
for (const node of osd_tree)
|
||||
{
|
||||
if ((levels[node.level] || node.level) < failure_domain_level)
|
||||
{
|
||||
flatten_tree(node.children||[], levels, failure_domain_level, osd_level, domains, i);
|
||||
}
|
||||
else
|
||||
{
|
||||
domains['dom'+(i.i++)] = extract_osds([ node ], levels, osd_level);
|
||||
}
|
||||
}
|
||||
return domains;
|
||||
}
|
||||
|
||||
function extract_osds(osd_tree, levels, osd_level, osds = {})
|
||||
{
|
||||
for (const node of osd_tree)
|
||||
{
|
||||
if ((levels[node.level] || node.level) >= osd_level)
|
||||
{
|
||||
osds[node.id] = node.size;
|
||||
}
|
||||
else
|
||||
{
|
||||
extract_osds(node.children||[], levels, osd_level, osds);
|
||||
}
|
||||
}
|
||||
return osds;
|
||||
}
|
||||
|
||||
// ordered = don't treat (x,y) and (y,x) as equal
|
||||
function random_combinations(osd_tree, pg_size, count, ordered)
|
||||
{
|
||||
let seed = 0x5f020e43;
|
||||
let rng = () =>
|
||||
{
|
||||
seed ^= seed << 13;
|
||||
seed ^= seed >> 17;
|
||||
seed ^= seed << 5;
|
||||
return seed + 2147483648;
|
||||
};
|
||||
const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
|
||||
const hosts = Object.keys(osd_tree).sort().filter(h => osds[h].length > 0);
|
||||
const r = {};
|
||||
// Generate random combinations including each OSD at least once
|
||||
for (let h = 0; h < hosts.length; h++)
|
||||
{
|
||||
for (let o = 0; o < osds[hosts[h]].length; o++)
|
||||
{
|
||||
const pg = [ osds[hosts[h]][o] ];
|
||||
const cur_hosts = [ ...hosts ];
|
||||
cur_hosts.splice(h, 1);
|
||||
for (let i = 1; i < pg_size && i < hosts.length; i++)
|
||||
{
|
||||
const next_host = rng() % cur_hosts.length;
|
||||
const next_osd = rng() % osds[cur_hosts[next_host]].length;
|
||||
pg.push(osds[cur_hosts[next_host]][next_osd]);
|
||||
cur_hosts.splice(next_host, 1);
|
||||
}
|
||||
while (pg.length < pg_size)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
}
|
||||
}
|
||||
// Generate purely random combinations
|
||||
while (count > 0)
|
||||
{
|
||||
let host_idx = [];
|
||||
const cur_hosts = [ ...hosts.map((h, i) => i) ];
|
||||
const max_hosts = pg_size < hosts.length ? pg_size : hosts.length;
|
||||
if (ordered)
|
||||
{
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
const r = rng() % cur_hosts.length;
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(r, 1);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
for (let i = 0; i < max_hosts; i++)
|
||||
{
|
||||
const r = rng() % (cur_hosts.length - (max_hosts - i - 1));
|
||||
host_idx[i] = cur_hosts[r];
|
||||
cur_hosts.splice(0, r+1);
|
||||
}
|
||||
}
|
||||
let pg = host_idx.map(h => osds[hosts[h]][rng() % osds[hosts[h]].length]);
|
||||
while (pg.length < pg_size)
|
||||
{
|
||||
pg.push(NO_OSD);
|
||||
}
|
||||
r['pg_'+pg.join('_')] = pg;
|
||||
count--;
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
// Super-stupid algorithm. Given the current OSD tree, generate all possible OSD combinations
|
||||
// osd_tree = { failure_domain1: { osd1: size1, ... }, ... }
|
||||
// ordered = return combinations without duplicates having different order
|
||||
function all_combinations(osd_tree, pg_size, ordered, count)
|
||||
{
|
||||
const hosts = Object.keys(osd_tree).sort();
|
||||
const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
|
||||
while (hosts.length < pg_size)
|
||||
{
|
||||
osds[NO_OSD] = [ NO_OSD ];
|
||||
hosts.push(NO_OSD);
|
||||
}
|
||||
let host_idx = [];
|
||||
let osd_idx = [];
|
||||
for (let i = 0; i < pg_size; i++)
|
||||
{
|
||||
host_idx.push(i);
|
||||
osd_idx.push(0);
|
||||
}
|
||||
const r = [];
|
||||
while (!count || count < 0 || r.length < count)
|
||||
{
|
||||
r.push(host_idx.map((hi, i) => osds[hosts[hi]][osd_idx[i]]));
|
||||
let inc = pg_size-1;
|
||||
while (inc >= 0)
|
||||
{
|
||||
osd_idx[inc]++;
|
||||
if (osd_idx[inc] >= osds[hosts[host_idx[inc]]].length)
|
||||
{
|
||||
osd_idx[inc] = 0;
|
||||
inc--;
|
||||
}
|
||||
else
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (inc < 0)
|
||||
{
|
||||
// no osds left in the current host combination, select the next one
|
||||
inc = pg_size-1;
|
||||
same_again: while (inc >= 0)
|
||||
{
|
||||
host_idx[inc]++;
|
||||
for (let prev_host = 0; prev_host < inc; prev_host++)
|
||||
{
|
||||
if (host_idx[prev_host] == host_idx[inc])
|
||||
{
|
||||
continue same_again;
|
||||
}
|
||||
}
|
||||
if (host_idx[inc] < (ordered ? hosts.length-(pg_size-1-inc) : hosts.length))
|
||||
{
|
||||
while ((++inc) < pg_size)
|
||||
{
|
||||
host_idx[inc] = (ordered ? host_idx[inc-1]+1 : 0);
|
||||
}
|
||||
break;
|
||||
}
|
||||
else
|
||||
{
|
||||
inc--;
|
||||
}
|
||||
}
|
||||
if (inc < 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return r;
|
||||
}
|
||||
|
||||
function check_combinations(osd_tree, pgs)
|
||||
{
|
||||
const hosts = Object.keys(osd_tree).sort();
|
||||
const host_per_osd = {};
|
||||
for (const host in osd_tree)
|
||||
{
|
||||
for (const osd in osd_tree[host])
|
||||
{
|
||||
host_per_osd[osd] = host;
|
||||
}
|
||||
}
|
||||
const res = [];
|
||||
skip_pg: for (const pg of pgs)
|
||||
{
|
||||
const seen_hosts = {};
|
||||
for (const osd of pg)
|
||||
{
|
||||
if (!host_per_osd[osd] || seen_hosts[host_per_osd[osd]])
|
||||
{
|
||||
continue skip_pg;
|
||||
}
|
||||
seen_hosts[host_per_osd[osd]] = true;
|
||||
}
|
||||
res.push(pg);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
function compat(params)
|
||||
{
|
||||
return {
|
||||
...params,
|
||||
osd_weights: Object.assign({}, ...Object.values(params.osd_tree)),
|
||||
combinator: new SimpleCombinator(params.osd_tree, params.pg_size, params.max_combinations||10000),
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
flatten_tree,
|
||||
SimpleCombinator,
|
||||
compat,
|
||||
NO_OSD,
|
||||
};
|
|
@ -7,6 +7,7 @@
|
|||
// This leads to really uneven OSD fill ratio in Ceph even when PGs are perfectly balanced.
|
||||
// But we support this case with the "parity_space" parameter in optimize_initial()/optimize_change().
|
||||
|
||||
const { SimpleCombinator } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
const osd_tree = {
|
||||
|
@ -114,16 +115,17 @@ Fine, let's try to optimize for it.
|
|||
|
||||
async function run()
|
||||
{
|
||||
const all_weights = Object.assign({}, ...Object.values(osd_tree));
|
||||
const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const eff = LPOptimizer.pg_list_space_efficiency(prev_pgs, all_weights, 2, 2.26);
|
||||
const osd_weights = Object.assign({}, ...Object.values(osd_tree));
|
||||
const total_weight = Object.values(osd_weights).reduce((a, c) => Number(a) + Number(c), 0);
|
||||
const eff = LPOptimizer.pg_list_space_efficiency(prev_pgs, osd_weights, 2, 2.26);
|
||||
const orig = eff*4.26 / total_weight;
|
||||
console.log('Original efficiency was: '+Math.round(orig*10000)/100+' %');
|
||||
|
||||
let prev = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 256, parity_space: 2.26 });
|
||||
const combinator = new SimpleCombinator(osd_tree, 3, 10000);
|
||||
let prev = await LPOptimizer.optimize_initial({ osd_weights, combinator, pg_size: 3, pg_count: 256, parity_space: 2.26 });
|
||||
LPOptimizer.print_change_stats(prev);
|
||||
|
||||
let next = await LPOptimizer.optimize_change({ prev_pgs, osd_tree, pg_size: 3, max_combinations: 10000, parity_space: 2.26 });
|
||||
let next = await LPOptimizer.optimize_change({ prev_pgs, osd_weights, combinator, pg_size: 3, parity_space: 2.26 });
|
||||
LPOptimizer.print_change_stats(next);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const { compat } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
async function run()
|
||||
|
@ -14,26 +15,26 @@ async function run()
|
|||
let res;
|
||||
|
||||
console.log('16 PGs, size=3');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16, ordered: false });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree, pg_size: 3, pg_count: 16, ordered: false }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 3, 'Initial distribution');
|
||||
console.log('\nChange size to 2');
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: false });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: false }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space >= 3*14/16 && res.osd_differs == 0, 'Redistribution');
|
||||
console.log('\nRemove OSD 3');
|
||||
const no3_tree = { ...osd_tree };
|
||||
delete no3_tree['300'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: no3_tree, pg_size: 2, ordered: false });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: no3_tree, pg_size: 2, ordered: false }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 2, 'Redistribution after OSD removal');
|
||||
|
||||
console.log('\n16 PGs, size=3, ordered');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 16, ordered: true });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree, pg_size: 3, pg_count: 16, ordered: true }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 3, 'Initial distribution');
|
||||
console.log('\nChange size to 2, ordered');
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: true });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2, ordered: true }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space >= 3*14/16 && res.osd_differs < 8, 'Redistribution');
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const { compat, flatten_tree } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
const crush_tree = [
|
||||
|
@ -36,44 +37,44 @@ const crush_tree = [
|
|||
] },
|
||||
];
|
||||
|
||||
const osd_tree = LPOptimizer.flatten_tree(crush_tree, {}, 1, 3);
|
||||
const osd_tree = flatten_tree(crush_tree, {}, 1, 3);
|
||||
console.log(osd_tree);
|
||||
|
||||
async function run()
|
||||
{
|
||||
const cur_tree = {};
|
||||
console.log('Empty tree:');
|
||||
let res = await LPOptimizer.optimize_initial({ osd_tree: cur_tree, pg_size: 3, pg_count: 256 });
|
||||
let res = await LPOptimizer.optimize_initial(compat({ osd_tree: cur_tree, pg_size: 3, pg_count: 256 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 0);
|
||||
console.log('\nAdding 1st failure domain:');
|
||||
cur_tree['dom1'] = osd_tree['dom1'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 12 && res.total_space == 12);
|
||||
console.log('\nAdding 2nd failure domain:');
|
||||
cur_tree['dom2'] = osd_tree['dom2'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 24 && res.total_space == 24);
|
||||
console.log('\nAdding 3rd failure domain:');
|
||||
cur_tree['dom3'] = osd_tree['dom3'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 36 && res.total_space == 36);
|
||||
console.log('\nRemoving 3rd failure domain:');
|
||||
delete cur_tree['dom3'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 24 && res.total_space == 24);
|
||||
console.log('\nRemoving 2nd failure domain:');
|
||||
delete cur_tree['dom2'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 12 && res.total_space == 12);
|
||||
console.log('\nRemoving 1st failure domain:');
|
||||
delete cur_tree['dom1'];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
assert(res.space == 0);
|
||||
}
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const { compat } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
const osd_tree = {
|
||||
|
@ -20,13 +21,13 @@ async function run()
|
|||
{
|
||||
let res;
|
||||
console.log('256 PGs, 3+3 OSDs, size=2');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 2, pg_count: 256 });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree, pg_size: 2, pg_count: 256 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
|
||||
// Should NOT fail with the "unfeasible or unbounded" exception
|
||||
console.log('\nRemoving osd.2');
|
||||
delete osd_tree[100][2];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
}
|
||||
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const { compat, flatten_tree } = require('./simple_pgs.js');
|
||||
const LPOptimizer = require('./lp-optimizer.js');
|
||||
|
||||
const osd_tree = {
|
||||
|
@ -84,31 +85,31 @@ async function run()
|
|||
// Space efficiency is ~99% in all cases.
|
||||
|
||||
console.log('256 PGs, size=2');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 2, pg_count: 256 });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree, pg_size: 2, pg_count: 256 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
console.log('\nAdding osd.8');
|
||||
osd_tree[500][8] = 3.58589;
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
console.log('\nRemoving osd.8');
|
||||
delete osd_tree[500][8];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 2 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
|
||||
console.log('\n256 PGs, size=3');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree, pg_size: 3, pg_count: 256 });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree, pg_size: 3, pg_count: 256 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
console.log('\nAdding osd.8');
|
||||
osd_tree[500][8] = 3.58589;
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
console.log('\nRemoving osd.8');
|
||||
delete osd_tree[500][8];
|
||||
res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 });
|
||||
res = await LPOptimizer.optimize_change(compat({ prev_pgs: res.int_pgs, osd_tree, pg_size: 3 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
|
||||
console.log('\n256 PGs, size=3, failure domain=rack');
|
||||
res = await LPOptimizer.optimize_initial({ osd_tree: LPOptimizer.flatten_tree(crush_tree, {}, 1, 3), pg_size: 3, pg_count: 256 });
|
||||
res = await LPOptimizer.optimize_initial(compat({ osd_tree: flatten_tree(crush_tree, {}, 1, 3), pg_size: 3, pg_count: 256 }));
|
||||
LPOptimizer.print_change_stats(res, false);
|
||||
}
|
||||
|
||||
|
|
|
@ -0,0 +1,104 @@
|
|||
const { random_custom_combinations, index_tree, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
|
||||
|
||||
function check(result, expected)
|
||||
{
|
||||
console.dir(result, { depth: null });
|
||||
if (JSON.stringify(result) !== JSON.stringify(expected))
|
||||
{
|
||||
process.stderr.write('Unexpected value, expected: ');
|
||||
console.dir(expected, { depth: null });
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
check(
|
||||
parse_pg_dsl("any, dc=1 host!=1, dc!=1, dc=3 host!=3, dc!=(1,3), dc=5 host!=5"),
|
||||
[
|
||||
[],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', 1 ] ],
|
||||
[ [ 'dc', '!=', 1 ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', 3 ] ],
|
||||
[ [ 'dc', '!=', [ 1, 3 ] ] ],
|
||||
[ [ 'dc', '=', 5 ], [ 'host', '!=', 5 ] ],
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
parse_pg_dsl("dc=meow, dc!=1, dc>2"),
|
||||
[
|
||||
[ [ 'dc', '=', { id: 'meow' } ] ],
|
||||
[ [ 'dc', '!=', 1 ] ],
|
||||
[ [ 'dc', '>', 2 ] ],
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
parse_level_indexes({ dc: '112233', host: 'ABCDEF' }),
|
||||
[
|
||||
[],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1 ] ], [ 'host', '!=', [ 1, 2 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
|
||||
[ [ 'dc', '=', 5 ], [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
parse_level_indexes({ dc: '112233', host: 'ABCDEF' }, [ 'dc', 'host' ]),
|
||||
[
|
||||
[],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1, 3 ] ] ],
|
||||
[ [ 'dc', '=', 5 ], [ 'host', '!=', [ 5 ] ] ],
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }),
|
||||
[
|
||||
[],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1 ] ], [ 'host', '!=', [ 1, 2 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3 ] ] ],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 3, 4 ] ] ],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 3, 4, 5 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1, 3 ] ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] ] ],
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
parse_level_indexes({ dc: '112211223333', host: '123456789ABC' }, [ 'dc', 'host' ]),
|
||||
[
|
||||
[],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3 ] ] ],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2 ] ] ],
|
||||
[ [ 'dc', '=', 1 ], [ 'host', '!=', [ 1, 2, 5 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4 ] ] ],
|
||||
[ [ 'dc', '=', 3 ], [ 'host', '!=', [ 3, 4, 7 ] ] ],
|
||||
[ [ 'dc', '!=', [ 1, 3 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10 ] ] ],
|
||||
[ [ 'dc', '=', 9 ], [ 'host', '!=', [ 9, 10, 11 ] ] ]
|
||||
]
|
||||
);
|
||||
|
||||
check(
|
||||
Object.keys(random_custom_combinations(index_tree([
|
||||
{ id: '1', size: 1, level: 'osd' },
|
||||
{ id: '2', size: 2, level: 'osd' },
|
||||
{ id: '3', size: 3, level: 'osd' }
|
||||
]), parse_level_indexes({ osd: '12' }), 10000)).sort(),
|
||||
[ 'pg_1_2', 'pg_1_3', 'pg_2_3' ]
|
||||
);
|
||||
|
||||
console.log('OK');
|
|
@ -50,7 +50,7 @@ from cinder.volume import configuration
|
|||
from cinder.volume import driver
|
||||
from cinder.volume import volume_utils
|
||||
|
||||
VERSION = '1.4.7'
|
||||
VERSION = '1.5.0'
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
|
||||
|
|
|
@ -0,0 +1,692 @@
|
|||
commit d85024bd803b3b91f15578ed22de4ce31856626f
|
||||
Author: Vitaliy Filippov <vitalif@yourcmc.ru>
|
||||
Date: Wed Jan 24 18:07:43 2024 +0300
|
||||
|
||||
Add Vitastor support
|
||||
|
||||
diff --git a/docs/schemas/domaincommon.rng b/docs/schemas/domaincommon.rng
|
||||
index 7fa5c2b8b5..2d77f391e7 100644
|
||||
--- a/docs/schemas/domaincommon.rng
|
||||
+++ b/docs/schemas/domaincommon.rng
|
||||
@@ -1898,6 +1898,35 @@
|
||||
</element>
|
||||
</define>
|
||||
|
||||
+ <define name="diskSourceNetworkProtocolVitastor">
|
||||
+ <element name="source">
|
||||
+ <interleave>
|
||||
+ <attribute name="protocol">
|
||||
+ <value>vitastor</value>
|
||||
+ </attribute>
|
||||
+ <ref name="diskSourceCommon"/>
|
||||
+ <optional>
|
||||
+ <attribute name="name"/>
|
||||
+ </optional>
|
||||
+ <optional>
|
||||
+ <attribute name="query"/>
|
||||
+ </optional>
|
||||
+ <zeroOrMore>
|
||||
+ <ref name="diskSourceNetworkHost"/>
|
||||
+ </zeroOrMore>
|
||||
+ <optional>
|
||||
+ <element name="config">
|
||||
+ <attribute name="file">
|
||||
+ <ref name="absFilePath"/>
|
||||
+ </attribute>
|
||||
+ <empty/>
|
||||
+ </element>
|
||||
+ </optional>
|
||||
+ <empty/>
|
||||
+ </interleave>
|
||||
+ </element>
|
||||
+ </define>
|
||||
+
|
||||
<define name="diskSourceNetworkProtocolISCSI">
|
||||
<element name="source">
|
||||
<attribute name="protocol">
|
||||
@@ -2154,6 +2183,7 @@
|
||||
<ref name="diskSourceNetworkProtocolSimple"/>
|
||||
<ref name="diskSourceNetworkProtocolVxHS"/>
|
||||
<ref name="diskSourceNetworkProtocolNFS"/>
|
||||
+ <ref name="diskSourceNetworkProtocolVitastor"/>
|
||||
</choice>
|
||||
</define>
|
||||
|
||||
diff --git a/include/libvirt/libvirt-storage.h b/include/libvirt/libvirt-storage.h
|
||||
index f89856b93e..a8cb9387e2 100644
|
||||
--- a/include/libvirt/libvirt-storage.h
|
||||
+++ b/include/libvirt/libvirt-storage.h
|
||||
@@ -246,6 +246,7 @@ typedef enum {
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_ZFS = 1 << 17,
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE = 1 << 18,
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_ISCSI_DIRECT = 1 << 19,
|
||||
+ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR = 1 << 20,
|
||||
} virConnectListAllStoragePoolsFlags;
|
||||
|
||||
int virConnectListAllStoragePools(virConnectPtr conn,
|
||||
diff --git a/src/conf/domain_conf.c b/src/conf/domain_conf.c
|
||||
index 5691b8d2d5..6669e8451d 100644
|
||||
--- a/src/conf/domain_conf.c
|
||||
+++ b/src/conf/domain_conf.c
|
||||
@@ -8293,7 +8293,8 @@ virDomainDiskSourceNetworkParse(xmlNodePtr node,
|
||||
src->configFile = virXPathString("string(./config/@file)", ctxt);
|
||||
|
||||
if (src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTP ||
|
||||
- src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS)
|
||||
+ src->protocol == VIR_STORAGE_NET_PROTOCOL_HTTPS ||
|
||||
+ src->protocol == VIR_STORAGE_NET_PROTOCOL_VITASTOR)
|
||||
src->query = virXMLPropString(node, "query");
|
||||
|
||||
if (virDomainStorageNetworkParseHosts(node, ctxt, &src->hosts, &src->nhosts) < 0)
|
||||
@@ -31267,6 +31268,7 @@ virDomainStorageSourceTranslateSourcePool(virStorageSource *src,
|
||||
|
||||
case VIR_STORAGE_POOL_MPATH:
|
||||
case VIR_STORAGE_POOL_RBD:
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
case VIR_STORAGE_POOL_SHEEPDOG:
|
||||
case VIR_STORAGE_POOL_GLUSTER:
|
||||
case VIR_STORAGE_POOL_LAST:
|
||||
diff --git a/src/conf/domain_validate.c b/src/conf/domain_validate.c
|
||||
index a4271f1247..621c1b7b31 100644
|
||||
--- a/src/conf/domain_validate.c
|
||||
+++ b/src/conf/domain_validate.c
|
||||
@@ -508,7 +508,7 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
|
||||
}
|
||||
}
|
||||
|
||||
- /* internal snapshots and config files are currently supported only with rbd: */
|
||||
+ /* internal snapshots are currently supported only with rbd: */
|
||||
if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
|
||||
src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD) {
|
||||
if (src->snapshot) {
|
||||
@@ -517,11 +517,15 @@ virDomainDiskDefValidateSourceChainOne(const virStorageSource *src)
|
||||
"only with 'rbd' disks"));
|
||||
return -1;
|
||||
}
|
||||
-
|
||||
+ }
|
||||
+ /* config files are currently supported only with rbd and vitastor: */
|
||||
+ if (virStorageSourceGetActualType(src) != VIR_STORAGE_TYPE_NETWORK &&
|
||||
+ src->protocol != VIR_STORAGE_NET_PROTOCOL_RBD &&
|
||||
+ src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR) {
|
||||
if (src->configFile) {
|
||||
virReportError(VIR_ERR_XML_ERROR, "%s",
|
||||
_("<config> element is currently supported "
|
||||
- "only with 'rbd' disks"));
|
||||
+ "only with 'rbd' and 'vitastor' disks"));
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
diff --git a/src/conf/storage_conf.c b/src/conf/storage_conf.c
|
||||
index 6690d26ffd..2255df9d28 100644
|
||||
--- a/src/conf/storage_conf.c
|
||||
+++ b/src/conf/storage_conf.c
|
||||
@@ -60,7 +60,7 @@ VIR_ENUM_IMPL(virStoragePool,
|
||||
"logical", "disk", "iscsi",
|
||||
"iscsi-direct", "scsi", "mpath",
|
||||
"rbd", "sheepdog", "gluster",
|
||||
- "zfs", "vstorage",
|
||||
+ "zfs", "vstorage", "vitastor",
|
||||
);
|
||||
|
||||
VIR_ENUM_IMPL(virStoragePoolFormatFileSystem,
|
||||
@@ -246,6 +246,18 @@ static virStoragePoolTypeInfo poolTypeInfo[] = {
|
||||
.formatToString = virStorageFileFormatTypeToString,
|
||||
}
|
||||
},
|
||||
+ {.poolType = VIR_STORAGE_POOL_VITASTOR,
|
||||
+ .poolOptions = {
|
||||
+ .flags = (VIR_STORAGE_POOL_SOURCE_HOST |
|
||||
+ VIR_STORAGE_POOL_SOURCE_NETWORK |
|
||||
+ VIR_STORAGE_POOL_SOURCE_NAME),
|
||||
+ },
|
||||
+ .volOptions = {
|
||||
+ .defaultFormat = VIR_STORAGE_FILE_RAW,
|
||||
+ .formatFromString = virStorageVolumeFormatFromString,
|
||||
+ .formatToString = virStorageFileFormatTypeToString,
|
||||
+ }
|
||||
+ },
|
||||
{.poolType = VIR_STORAGE_POOL_SHEEPDOG,
|
||||
.poolOptions = {
|
||||
.flags = (VIR_STORAGE_POOL_SOURCE_HOST |
|
||||
@@ -546,6 +558,11 @@ virStoragePoolDefParseSource(xmlXPathContextPtr ctxt,
|
||||
_("element 'name' is mandatory for RBD pool"));
|
||||
return -1;
|
||||
}
|
||||
+ if (pool_type == VIR_STORAGE_POOL_VITASTOR && source->name == NULL) {
|
||||
+ virReportError(VIR_ERR_XML_ERROR, "%s",
|
||||
+ _("element 'name' is mandatory for Vitastor pool"));
|
||||
+ return -1;
|
||||
+ }
|
||||
|
||||
if (options->formatFromString) {
|
||||
g_autofree char *format = NULL;
|
||||
@@ -1176,6 +1193,7 @@ virStoragePoolDefFormatBuf(virBuffer *buf,
|
||||
/* RBD, Sheepdog, Gluster and Iscsi-direct devices are not local block devs nor
|
||||
* files, so they don't have a target */
|
||||
if (def->type != VIR_STORAGE_POOL_RBD &&
|
||||
+ def->type != VIR_STORAGE_POOL_VITASTOR &&
|
||||
def->type != VIR_STORAGE_POOL_SHEEPDOG &&
|
||||
def->type != VIR_STORAGE_POOL_GLUSTER &&
|
||||
def->type != VIR_STORAGE_POOL_ISCSI_DIRECT) {
|
||||
diff --git a/src/conf/storage_conf.h b/src/conf/storage_conf.h
|
||||
index aaecf138d6..97172db38b 100644
|
||||
--- a/src/conf/storage_conf.h
|
||||
+++ b/src/conf/storage_conf.h
|
||||
@@ -106,6 +106,7 @@ typedef enum {
|
||||
VIR_STORAGE_POOL_GLUSTER, /* Gluster device */
|
||||
VIR_STORAGE_POOL_ZFS, /* ZFS */
|
||||
VIR_STORAGE_POOL_VSTORAGE, /* Virtuozzo Storage */
|
||||
+ VIR_STORAGE_POOL_VITASTOR, /* Vitastor */
|
||||
|
||||
VIR_STORAGE_POOL_LAST,
|
||||
} virStoragePoolType;
|
||||
@@ -466,6 +467,7 @@ VIR_ENUM_DECL(virStoragePartedFs);
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_SCSI | \
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_MPATH | \
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_RBD | \
|
||||
+ VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR | \
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG | \
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER | \
|
||||
VIR_CONNECT_LIST_STORAGE_POOLS_ZFS | \
|
||||
diff --git a/src/conf/storage_source_conf.c b/src/conf/storage_source_conf.c
|
||||
index d42f715f26..29d8da3d10 100644
|
||||
--- a/src/conf/storage_source_conf.c
|
||||
+++ b/src/conf/storage_source_conf.c
|
||||
@@ -86,6 +86,7 @@ VIR_ENUM_IMPL(virStorageNetProtocol,
|
||||
"ssh",
|
||||
"vxhs",
|
||||
"nfs",
|
||||
+ "vitastor",
|
||||
);
|
||||
|
||||
|
||||
@@ -1265,6 +1266,7 @@ virStorageSourceNetworkDefaultPort(virStorageNetProtocol protocol)
|
||||
case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
|
||||
return 24007;
|
||||
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
/* we don't provide a default for RBD */
|
||||
return 0;
|
||||
diff --git a/src/conf/storage_source_conf.h b/src/conf/storage_source_conf.h
|
||||
index c4a026881c..67568e9181 100644
|
||||
--- a/src/conf/storage_source_conf.h
|
||||
+++ b/src/conf/storage_source_conf.h
|
||||
@@ -128,6 +128,7 @@ typedef enum {
|
||||
VIR_STORAGE_NET_PROTOCOL_SSH,
|
||||
VIR_STORAGE_NET_PROTOCOL_VXHS,
|
||||
VIR_STORAGE_NET_PROTOCOL_NFS,
|
||||
+ VIR_STORAGE_NET_PROTOCOL_VITASTOR,
|
||||
|
||||
VIR_STORAGE_NET_PROTOCOL_LAST
|
||||
} virStorageNetProtocol;
|
||||
diff --git a/src/conf/virstorageobj.c b/src/conf/virstorageobj.c
|
||||
index 02903ac487..504df599fb 100644
|
||||
--- a/src/conf/virstorageobj.c
|
||||
+++ b/src/conf/virstorageobj.c
|
||||
@@ -1481,6 +1481,7 @@ virStoragePoolObjSourceFindDuplicateCb(const void *payload,
|
||||
return 1;
|
||||
break;
|
||||
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
case VIR_STORAGE_POOL_RBD:
|
||||
case VIR_STORAGE_POOL_LAST:
|
||||
break;
|
||||
@@ -1978,6 +1979,8 @@ virStoragePoolObjMatch(virStoragePoolObj *obj,
|
||||
(obj->def->type == VIR_STORAGE_POOL_MPATH)) ||
|
||||
(MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_RBD) &&
|
||||
(obj->def->type == VIR_STORAGE_POOL_RBD)) ||
|
||||
+ (MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR) &&
|
||||
+ (obj->def->type == VIR_STORAGE_POOL_VITASTOR)) ||
|
||||
(MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG) &&
|
||||
(obj->def->type == VIR_STORAGE_POOL_SHEEPDOG)) ||
|
||||
(MATCH(VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER) &&
|
||||
diff --git a/src/libvirt-storage.c b/src/libvirt-storage.c
|
||||
index cbc522b300..b4760fa58d 100644
|
||||
--- a/src/libvirt-storage.c
|
||||
+++ b/src/libvirt-storage.c
|
||||
@@ -92,6 +92,7 @@ virStoragePoolGetConnect(virStoragePoolPtr pool)
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_SCSI
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_MPATH
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_RBD
|
||||
+ * VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_SHEEPDOG
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_GLUSTER
|
||||
* VIR_CONNECT_LIST_STORAGE_POOLS_ZFS
|
||||
diff --git a/src/libxl/libxl_conf.c b/src/libxl/libxl_conf.c
|
||||
index 1ac6253ad7..abe4587f94 100644
|
||||
--- a/src/libxl/libxl_conf.c
|
||||
+++ b/src/libxl/libxl_conf.c
|
||||
@@ -962,6 +962,7 @@ libxlMakeNetworkDiskSrcStr(virStorageSource *src,
|
||||
case VIR_STORAGE_NET_PROTOCOL_SSH:
|
||||
case VIR_STORAGE_NET_PROTOCOL_VXHS:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NFS:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_LAST:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
virReportError(VIR_ERR_NO_SUPPORT,
|
||||
diff --git a/src/libxl/xen_xl.c b/src/libxl/xen_xl.c
|
||||
index 7604e3d534..6453bb9776 100644
|
||||
--- a/src/libxl/xen_xl.c
|
||||
+++ b/src/libxl/xen_xl.c
|
||||
@@ -1506,6 +1506,7 @@ xenFormatXLDiskSrcNet(virStorageSource *src)
|
||||
case VIR_STORAGE_NET_PROTOCOL_SSH:
|
||||
case VIR_STORAGE_NET_PROTOCOL_VXHS:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NFS:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_LAST:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
virReportError(VIR_ERR_NO_SUPPORT,
|
||||
diff --git a/src/qemu/qemu_block.c b/src/qemu/qemu_block.c
|
||||
index e5ff653a60..884ecc79ea 100644
|
||||
--- a/src/qemu/qemu_block.c
|
||||
+++ b/src/qemu/qemu_block.c
|
||||
@@ -943,6 +943,38 @@ qemuBlockStorageSourceGetRBDProps(virStorageSource *src,
|
||||
}
|
||||
|
||||
|
||||
+static virJSONValue *
|
||||
+qemuBlockStorageSourceGetVitastorProps(virStorageSource *src)
|
||||
+{
|
||||
+ virJSONValue *ret = NULL;
|
||||
+ virStorageNetHostDef *host;
|
||||
+ size_t i;
|
||||
+ g_auto(virBuffer) buf = VIR_BUFFER_INITIALIZER;
|
||||
+ g_autofree char *etcd = NULL;
|
||||
+
|
||||
+ for (i = 0; i < src->nhosts; i++) {
|
||||
+ host = src->hosts + i;
|
||||
+ if ((virStorageNetHostTransport)host->transport != VIR_STORAGE_NET_HOST_TRANS_TCP) {
|
||||
+ return NULL;
|
||||
+ }
|
||||
+ virBufferAsprintf(&buf, i > 0 ? ",%s:%u" : "%s:%u", host->name, host->port);
|
||||
+ }
|
||||
+ if (src->nhosts > 0) {
|
||||
+ etcd = virBufferContentAndReset(&buf);
|
||||
+ }
|
||||
+
|
||||
+ if (virJSONValueObjectCreate(&ret,
|
||||
+ "S:etcd-host", etcd,
|
||||
+ "S:etcd-prefix", src->query,
|
||||
+ "S:config-path", src->configFile,
|
||||
+ "s:image", src->path,
|
||||
+ NULL) < 0)
|
||||
+ return NULL;
|
||||
+
|
||||
+ return ret;
|
||||
+}
|
||||
+
|
||||
+
|
||||
static virJSONValue *
|
||||
qemuBlockStorageSourceGetSheepdogProps(virStorageSource *src)
|
||||
{
|
||||
@@ -1233,6 +1265,12 @@ qemuBlockStorageSourceGetBackendProps(virStorageSource *src,
|
||||
return NULL;
|
||||
break;
|
||||
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
+ driver = "vitastor";
|
||||
+ if (!(fileprops = qemuBlockStorageSourceGetVitastorProps(src)))
|
||||
+ return NULL;
|
||||
+ break;
|
||||
+
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
driver = "sheepdog";
|
||||
if (!(fileprops = qemuBlockStorageSourceGetSheepdogProps(src)))
|
||||
@@ -2244,6 +2282,7 @@ qemuBlockGetBackingStoreString(virStorageSource *src,
|
||||
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_VXHS:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NFS:
|
||||
case VIR_STORAGE_NET_PROTOCOL_SSH:
|
||||
@@ -2626,6 +2665,12 @@ qemuBlockStorageSourceCreateGetStorageProps(virStorageSource *src,
|
||||
return -1;
|
||||
break;
|
||||
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
+ driver = "vitastor";
|
||||
+ if (!(location = qemuBlockStorageSourceGetVitastorProps(src)))
|
||||
+ return -1;
|
||||
+ break;
|
||||
+
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
driver = "sheepdog";
|
||||
if (!(location = qemuBlockStorageSourceGetSheepdogProps(src)))
|
||||
diff --git a/src/qemu/qemu_command.c b/src/qemu/qemu_command.c
|
||||
index d822533ccb..afe2087303 100644
|
||||
--- a/src/qemu/qemu_command.c
|
||||
+++ b/src/qemu/qemu_command.c
|
||||
@@ -1723,6 +1723,43 @@ qemuBuildNetworkDriveStr(virStorageSource *src,
|
||||
ret = virBufferContentAndReset(&buf);
|
||||
break;
|
||||
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
+ if (strchr(src->path, ':')) {
|
||||
+ virReportError(VIR_ERR_CONFIG_UNSUPPORTED,
|
||||
+ _("':' not allowed in Vitastor source volume name '%s'"),
|
||||
+ src->path);
|
||||
+ return NULL;
|
||||
+ }
|
||||
+
|
||||
+ virBufferStrcat(&buf, "vitastor:image=", src->path, NULL);
|
||||
+
|
||||
+ if (src->nhosts > 0) {
|
||||
+ virBufferAddLit(&buf, ":etcd-host=");
|
||||
+ for (i = 0; i < src->nhosts; i++) {
|
||||
+ if (i)
|
||||
+ virBufferAddLit(&buf, ",");
|
||||
+
|
||||
+ /* assume host containing : is ipv6 */
|
||||
+ if (strchr(src->hosts[i].name, ':'))
|
||||
+ virBufferEscape(&buf, '\\', ":", "[%s]",
|
||||
+ src->hosts[i].name);
|
||||
+ else
|
||||
+ virBufferAsprintf(&buf, "%s", src->hosts[i].name);
|
||||
+
|
||||
+ if (src->hosts[i].port)
|
||||
+ virBufferAsprintf(&buf, "\\:%u", src->hosts[i].port);
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if (src->configFile)
|
||||
+ virBufferEscape(&buf, '\\', ":", ":config-path=%s", src->configFile);
|
||||
+
|
||||
+ if (src->query)
|
||||
+ virBufferEscape(&buf, '\\', ":", ":etcd-prefix=%s", src->query);
|
||||
+
|
||||
+ ret = virBufferContentAndReset(&buf);
|
||||
+ break;
|
||||
+
|
||||
case VIR_STORAGE_NET_PROTOCOL_VXHS:
|
||||
virReportError(VIR_ERR_INTERNAL_ERROR, "%s",
|
||||
_("VxHS protocol does not support URI syntax"));
|
||||
diff --git a/src/qemu/qemu_domain.c b/src/qemu/qemu_domain.c
|
||||
index a8401bac30..3dc1fe6db0 100644
|
||||
--- a/src/qemu/qemu_domain.c
|
||||
+++ b/src/qemu/qemu_domain.c
|
||||
@@ -4731,7 +4731,8 @@ qemuDomainValidateStorageSource(virStorageSource *src,
|
||||
if (src->query &&
|
||||
(actualType != VIR_STORAGE_TYPE_NETWORK ||
|
||||
(src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTPS &&
|
||||
- src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP))) {
|
||||
+ src->protocol != VIR_STORAGE_NET_PROTOCOL_HTTP &&
|
||||
+ src->protocol != VIR_STORAGE_NET_PROTOCOL_VITASTOR))) {
|
||||
virReportError(VIR_ERR_CONFIG_UNSUPPORTED, "%s",
|
||||
_("query is supported only with HTTP(S) protocols"));
|
||||
return -1;
|
||||
@@ -9919,6 +9920,7 @@ qemuDomainPrepareStorageSourceTLS(virStorageSource *src,
|
||||
break;
|
||||
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
|
||||
case VIR_STORAGE_NET_PROTOCOL_ISCSI:
|
||||
diff --git a/src/qemu/qemu_snapshot.c b/src/qemu/qemu_snapshot.c
|
||||
index f92e00f9c0..854a3fbc90 100644
|
||||
--- a/src/qemu/qemu_snapshot.c
|
||||
+++ b/src/qemu/qemu_snapshot.c
|
||||
@@ -393,6 +393,7 @@ qemuSnapshotPrepareDiskExternalInactive(virDomainSnapshotDiskDef *snapdisk,
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NBD:
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
|
||||
case VIR_STORAGE_NET_PROTOCOL_ISCSI:
|
||||
@@ -485,6 +486,7 @@ qemuSnapshotPrepareDiskExternalActive(virDomainObj *vm,
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NBD:
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_ISCSI:
|
||||
case VIR_STORAGE_NET_PROTOCOL_HTTP:
|
||||
@@ -638,6 +640,7 @@ qemuSnapshotPrepareDiskInternal(virDomainDiskDef *disk,
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NBD:
|
||||
case VIR_STORAGE_NET_PROTOCOL_RBD:
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_GLUSTER:
|
||||
case VIR_STORAGE_NET_PROTOCOL_ISCSI:
|
||||
diff --git a/src/storage/storage_driver.c b/src/storage/storage_driver.c
|
||||
index 4df2c75a2b..5a5e48ef71 100644
|
||||
--- a/src/storage/storage_driver.c
|
||||
+++ b/src/storage/storage_driver.c
|
||||
@@ -1643,6 +1643,7 @@ storageVolLookupByPathCallback(virStoragePoolObj *obj,
|
||||
|
||||
case VIR_STORAGE_POOL_GLUSTER:
|
||||
case VIR_STORAGE_POOL_RBD:
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
case VIR_STORAGE_POOL_SHEEPDOG:
|
||||
case VIR_STORAGE_POOL_ZFS:
|
||||
case VIR_STORAGE_POOL_LAST:
|
||||
diff --git a/src/storage_file/storage_source_backingstore.c b/src/storage_file/storage_source_backingstore.c
|
||||
index e48ae725ab..2017ccc88c 100644
|
||||
--- a/src/storage_file/storage_source_backingstore.c
|
||||
+++ b/src/storage_file/storage_source_backingstore.c
|
||||
@@ -284,6 +284,75 @@ virStorageSourceParseRBDColonString(const char *rbdstr,
|
||||
}
|
||||
|
||||
|
||||
+static int
|
||||
+virStorageSourceParseVitastorColonString(const char *colonstr,
|
||||
+ virStorageSource *src)
|
||||
+{
|
||||
+ char *p, *e, *next;
|
||||
+ g_autofree char *options = NULL;
|
||||
+
|
||||
+ /* optionally skip the "vitastor:" prefix if provided */
|
||||
+ if (STRPREFIX(colonstr, "vitastor:"))
|
||||
+ colonstr += strlen("vitastor:");
|
||||
+
|
||||
+ options = g_strdup(colonstr);
|
||||
+
|
||||
+ p = options;
|
||||
+ while (*p) {
|
||||
+ /* find : delimiter or end of string */
|
||||
+ for (e = p; *e && *e != ':'; ++e) {
|
||||
+ if (*e == '\\') {
|
||||
+ e++;
|
||||
+ if (*e == '\0')
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+ if (*e == '\0') {
|
||||
+ next = e; /* last kv pair */
|
||||
+ } else {
|
||||
+ next = e + 1;
|
||||
+ *e = '\0';
|
||||
+ }
|
||||
+
|
||||
+ if (STRPREFIX(p, "image=")) {
|
||||
+ src->path = g_strdup(p + strlen("image="));
|
||||
+ } else if (STRPREFIX(p, "etcd-prefix=")) {
|
||||
+ src->query = g_strdup(p + strlen("etcd-prefix="));
|
||||
+ } else if (STRPREFIX(p, "config-path=")) {
|
||||
+ src->configFile = g_strdup(p + strlen("config-path="));
|
||||
+ } else if (STRPREFIX(p, "etcd-host=")) {
|
||||
+ char *h, *sep;
|
||||
+
|
||||
+ h = p + strlen("etcd-host=");
|
||||
+ while (h < e) {
|
||||
+ for (sep = h; sep < e; ++sep) {
|
||||
+ if (*sep == '\\' && (sep[1] == ',' ||
|
||||
+ sep[1] == ';' ||
|
||||
+ sep[1] == ' ')) {
|
||||
+ *sep = '\0';
|
||||
+ sep += 2;
|
||||
+ break;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ if (virStorageSourceRBDAddHost(src, h) < 0)
|
||||
+ return -1;
|
||||
+
|
||||
+ h = sep;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ p = next;
|
||||
+ }
|
||||
+
|
||||
+ if (!src->path) {
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
+
|
||||
static int
|
||||
virStorageSourceParseNBDColonString(const char *nbdstr,
|
||||
virStorageSource *src)
|
||||
@@ -396,6 +465,11 @@ virStorageSourceParseBackingColon(virStorageSource *src,
|
||||
return -1;
|
||||
break;
|
||||
|
||||
+ case VIR_STORAGE_NET_PROTOCOL_VITASTOR:
|
||||
+ if (virStorageSourceParseVitastorColonString(path, src) < 0)
|
||||
+ return -1;
|
||||
+ break;
|
||||
+
|
||||
case VIR_STORAGE_NET_PROTOCOL_SHEEPDOG:
|
||||
case VIR_STORAGE_NET_PROTOCOL_LAST:
|
||||
case VIR_STORAGE_NET_PROTOCOL_NONE:
|
||||
@@ -984,6 +1058,54 @@ virStorageSourceParseBackingJSONRBD(virStorageSource *src,
|
||||
return 0;
|
||||
}
|
||||
|
||||
+static int
|
||||
+virStorageSourceParseBackingJSONVitastor(virStorageSource *src,
|
||||
+ virJSONValue *json,
|
||||
+ const char *jsonstr G_GNUC_UNUSED,
|
||||
+ int opaque G_GNUC_UNUSED)
|
||||
+{
|
||||
+ const char *filename;
|
||||
+ const char *image = virJSONValueObjectGetString(json, "image");
|
||||
+ const char *conf = virJSONValueObjectGetString(json, "config-path");
|
||||
+ const char *etcd_prefix = virJSONValueObjectGetString(json, "etcd-prefix");
|
||||
+ virJSONValue *servers = virJSONValueObjectGetArray(json, "server");
|
||||
+ size_t nservers;
|
||||
+ size_t i;
|
||||
+
|
||||
+ src->type = VIR_STORAGE_TYPE_NETWORK;
|
||||
+ src->protocol = VIR_STORAGE_NET_PROTOCOL_VITASTOR;
|
||||
+
|
||||
+ /* legacy syntax passed via 'filename' option */
|
||||
+ if ((filename = virJSONValueObjectGetString(json, "filename")))
|
||||
+ return virStorageSourceParseVitastorColonString(filename, src);
|
||||
+
|
||||
+ if (!image) {
|
||||
+ virReportError(VIR_ERR_INVALID_ARG, "%s",
|
||||
+ _("missing image name in Vitastor backing volume "
|
||||
+ "JSON specification"));
|
||||
+ return -1;
|
||||
+ }
|
||||
+
|
||||
+ src->path = g_strdup(image);
|
||||
+ src->configFile = g_strdup(conf);
|
||||
+ src->query = g_strdup(etcd_prefix);
|
||||
+
|
||||
+ if (servers) {
|
||||
+ nservers = virJSONValueArraySize(servers);
|
||||
+
|
||||
+ src->hosts = g_new0(virStorageNetHostDef, nservers);
|
||||
+ src->nhosts = nservers;
|
||||
+
|
||||
+ for (i = 0; i < nservers; i++) {
|
||||
+ if (virStorageSourceParseBackingJSONInetSocketAddress(src->hosts + i,
|
||||
+ virJSONValueArrayGet(servers, i)) < 0)
|
||||
+ return -1;
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
+ return 0;
|
||||
+}
|
||||
+
|
||||
static int
|
||||
virStorageSourceParseBackingJSONRaw(virStorageSource *src,
|
||||
virJSONValue *json,
|
||||
@@ -1162,6 +1284,7 @@ static const struct virStorageSourceJSONDriverParser jsonParsers[] = {
|
||||
{"sheepdog", false, virStorageSourceParseBackingJSONSheepdog, 0},
|
||||
{"ssh", false, virStorageSourceParseBackingJSONSSH, 0},
|
||||
{"rbd", false, virStorageSourceParseBackingJSONRBD, 0},
|
||||
+ {"vitastor", false, virStorageSourceParseBackingJSONVitastor, 0},
|
||||
{"raw", true, virStorageSourceParseBackingJSONRaw, 0},
|
||||
{"nfs", false, virStorageSourceParseBackingJSONNFS, 0},
|
||||
{"vxhs", false, virStorageSourceParseBackingJSONVxHS, 0},
|
||||
diff --git a/src/test/test_driver.c b/src/test/test_driver.c
|
||||
index 0e93b79922..b4d33f5f56 100644
|
||||
--- a/src/test/test_driver.c
|
||||
+++ b/src/test/test_driver.c
|
||||
@@ -7367,6 +7367,7 @@ testStorageVolumeTypeForPool(int pooltype)
|
||||
case VIR_STORAGE_POOL_ISCSI_DIRECT:
|
||||
case VIR_STORAGE_POOL_GLUSTER:
|
||||
case VIR_STORAGE_POOL_RBD:
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
return VIR_STORAGE_VOL_NETWORK;
|
||||
case VIR_STORAGE_POOL_LOGICAL:
|
||||
case VIR_STORAGE_POOL_DISK:
|
||||
diff --git a/tests/storagepoolcapsschemadata/poolcaps-fs.xml b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
|
||||
index eee75af746..8bd0a57bdd 100644
|
||||
--- a/tests/storagepoolcapsschemadata/poolcaps-fs.xml
|
||||
+++ b/tests/storagepoolcapsschemadata/poolcaps-fs.xml
|
||||
@@ -204,4 +204,11 @@
|
||||
</enum>
|
||||
</volOptions>
|
||||
</pool>
|
||||
+ <pool type='vitastor' supported='no'>
|
||||
+ <volOptions>
|
||||
+ <defaultFormat type='raw'/>
|
||||
+ <enum name='targetFormatType'>
|
||||
+ </enum>
|
||||
+ </volOptions>
|
||||
+ </pool>
|
||||
</storagepoolCapabilities>
|
||||
diff --git a/tests/storagepoolcapsschemadata/poolcaps-full.xml b/tests/storagepoolcapsschemadata/poolcaps-full.xml
|
||||
index 805950a937..852df0de16 100644
|
||||
--- a/tests/storagepoolcapsschemadata/poolcaps-full.xml
|
||||
+++ b/tests/storagepoolcapsschemadata/poolcaps-full.xml
|
||||
@@ -204,4 +204,11 @@
|
||||
</enum>
|
||||
</volOptions>
|
||||
</pool>
|
||||
+ <pool type='vitastor' supported='yes'>
|
||||
+ <volOptions>
|
||||
+ <defaultFormat type='raw'/>
|
||||
+ <enum name='targetFormatType'>
|
||||
+ </enum>
|
||||
+ </volOptions>
|
||||
+ </pool>
|
||||
</storagepoolCapabilities>
|
||||
diff --git a/tests/storagepoolxml2argvtest.c b/tests/storagepoolxml2argvtest.c
|
||||
index 449b745519..7f95cc8e08 100644
|
||||
--- a/tests/storagepoolxml2argvtest.c
|
||||
+++ b/tests/storagepoolxml2argvtest.c
|
||||
@@ -68,6 +68,7 @@ testCompareXMLToArgvFiles(bool shouldFail,
|
||||
case VIR_STORAGE_POOL_GLUSTER:
|
||||
case VIR_STORAGE_POOL_ZFS:
|
||||
case VIR_STORAGE_POOL_VSTORAGE:
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
case VIR_STORAGE_POOL_LAST:
|
||||
default:
|
||||
VIR_TEST_DEBUG("pool type '%s' has no xml2argv test", defTypeStr);
|
||||
diff --git a/tools/virsh-pool.c b/tools/virsh-pool.c
|
||||
index d391257f6e..46799c4a90 100644
|
||||
--- a/tools/virsh-pool.c
|
||||
+++ b/tools/virsh-pool.c
|
||||
@@ -1213,6 +1213,9 @@ cmdPoolList(vshControl *ctl, const vshCmd *cmd G_GNUC_UNUSED)
|
||||
case VIR_STORAGE_POOL_VSTORAGE:
|
||||
flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VSTORAGE;
|
||||
break;
|
||||
+ case VIR_STORAGE_POOL_VITASTOR:
|
||||
+ flags |= VIR_CONNECT_LIST_STORAGE_POOLS_VITASTOR;
|
||||
+ break;
|
||||
case VIR_STORAGE_POOL_LAST:
|
||||
break;
|
||||
}
|
|
@ -24,4 +24,4 @@ rm fio
|
|||
mv fio-copy fio
|
||||
FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
|
||||
perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
|
||||
tar --transform 's#^#vitastor-1.4.7/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.4.7$(rpm --eval '%dist').tar.gz *
|
||||
tar --transform 's#^#vitastor-1.5.0/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-1.5.0$(rpm --eval '%dist').tar.gz *
|
||||
|
|
|
@ -36,7 +36,7 @@ ADD . /root/vitastor
|
|||
RUN set -e; \
|
||||
cd /root/vitastor/rpm; \
|
||||
sh build-tarball.sh; \
|
||||
cp /root/vitastor-1.4.7.el7.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp /root/vitastor-1.5.0.el7.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
rpmbuild -ba vitastor.spec; \
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
Name: vitastor
|
||||
Version: 1.4.7
|
||||
Version: 1.5.0
|
||||
Release: 1%{?dist}
|
||||
Summary: Vitastor, a fast software-defined clustered block storage
|
||||
|
||||
License: Vitastor Network Public License 1.1
|
||||
URL: https://vitastor.io/
|
||||
Source0: vitastor-1.4.7.el7.tar.gz
|
||||
Source0: vitastor-1.5.0.el7.tar.gz
|
||||
|
||||
BuildRequires: liburing-devel >= 0.6
|
||||
BuildRequires: gperftools-devel
|
||||
|
@ -149,9 +149,12 @@ mkdir -p /etc/vitastor
|
|||
%_bindir/vitastor-nfs
|
||||
%_bindir/vitastor-cli
|
||||
%_bindir/vitastor-rm
|
||||
%_bindir/vitastor-kv
|
||||
%_bindir/vitastor-kv-stress
|
||||
%_bindir/vita
|
||||
%_libdir/libvitastor_blk.so*
|
||||
%_libdir/libvitastor_client.so*
|
||||
%_libdir/libvitastor_kv.so*
|
||||
|
||||
|
||||
%files -n vitastor-client-devel
|
||||
|
|
|
@ -35,7 +35,7 @@ ADD . /root/vitastor
|
|||
RUN set -e; \
|
||||
cd /root/vitastor/rpm; \
|
||||
sh build-tarball.sh; \
|
||||
cp /root/vitastor-1.4.7.el8.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp /root/vitastor-1.5.0.el8.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
rpmbuild -ba vitastor.spec; \
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
Name: vitastor
|
||||
Version: 1.4.7
|
||||
Version: 1.5.0
|
||||
Release: 1%{?dist}
|
||||
Summary: Vitastor, a fast software-defined clustered block storage
|
||||
|
||||
License: Vitastor Network Public License 1.1
|
||||
URL: https://vitastor.io/
|
||||
Source0: vitastor-1.4.7.el8.tar.gz
|
||||
Source0: vitastor-1.5.0.el8.tar.gz
|
||||
|
||||
BuildRequires: liburing-devel >= 0.6
|
||||
BuildRequires: gperftools-devel
|
||||
|
@ -146,9 +146,12 @@ mkdir -p /etc/vitastor
|
|||
%_bindir/vitastor-nfs
|
||||
%_bindir/vitastor-cli
|
||||
%_bindir/vitastor-rm
|
||||
%_bindir/vitastor-kv
|
||||
%_bindir/vitastor-kv-stress
|
||||
%_bindir/vita
|
||||
%_libdir/libvitastor_blk.so*
|
||||
%_libdir/libvitastor_client.so*
|
||||
%_libdir/libvitastor_kv.so*
|
||||
|
||||
|
||||
%files -n vitastor-client-devel
|
||||
|
|
|
@ -18,7 +18,7 @@ ADD . /root/vitastor
|
|||
RUN set -e; \
|
||||
cd /root/vitastor/rpm; \
|
||||
sh build-tarball.sh; \
|
||||
cp /root/vitastor-1.4.7.el9.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp /root/vitastor-1.5.0.el9.tar.gz ~/rpmbuild/SOURCES; \
|
||||
cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
|
||||
cd ~/rpmbuild/SPECS/; \
|
||||
rpmbuild -ba vitastor.spec; \
|
||||
|
|
|
@ -1,11 +1,11 @@
|
|||
Name: vitastor
|
||||
Version: 1.4.7
|
||||
Version: 1.5.0
|
||||
Release: 1%{?dist}
|
||||
Summary: Vitastor, a fast software-defined clustered block storage
|
||||
|
||||
License: Vitastor Network Public License 1.1
|
||||
URL: https://vitastor.io/
|
||||
Source0: vitastor-1.4.7.el9.tar.gz
|
||||
Source0: vitastor-1.5.0.el9.tar.gz
|
||||
|
||||
BuildRequires: liburing-devel >= 0.6
|
||||
BuildRequires: gperftools-devel
|
||||
|
@ -139,9 +139,12 @@ mkdir -p /etc/vitastor
|
|||
%_bindir/vitastor-nfs
|
||||
%_bindir/vitastor-cli
|
||||
%_bindir/vitastor-rm
|
||||
%_bindir/vitastor-kv
|
||||
%_bindir/vitastor-kv-stress
|
||||
%_bindir/vita
|
||||
%_libdir/libvitastor_blk.so*
|
||||
%_libdir/libvitastor_client.so*
|
||||
%_libdir/libvitastor_kv.so*
|
||||
|
||||
|
||||
%files -n vitastor-client-devel
|
||||
|
|
|
@ -16,7 +16,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
|
|||
set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
|
||||
endif()
|
||||
|
||||
add_definitions(-DVERSION="1.4.7")
|
||||
add_definitions(-DVERSION="1.5.0")
|
||||
add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
|
||||
add_link_options(-fno-omit-frame-pointer)
|
||||
if (${WITH_ASAN})
|
||||
|
@ -25,15 +25,15 @@ if (${WITH_ASAN})
|
|||
endif (${WITH_ASAN})
|
||||
|
||||
set(CMAKE_BUILD_TYPE RelWithDebInfo)
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[12]?" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]O)[^ \t\r\n]*" "\\13" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_MINSIZEREL "${CMAKE_C_FLAGS_MINSIZEREL}")
|
||||
string(REGEX REPLACE "([\\/\\-]D) *NDEBUG" "" CMAKE_C_FLAGS_RELWITHDEBINFO "${CMAKE_C_FLAGS_RELWITHDEBINFO}")
|
||||
|
@ -145,7 +145,6 @@ add_library(vitastor_client SHARED
|
|||
cli_status.cpp
|
||||
cli_describe.cpp
|
||||
cli_fix.cpp
|
||||
cli_df.cpp
|
||||
cli_ls.cpp
|
||||
cli_create.cpp
|
||||
cli_modify.cpp
|
||||
|
@ -154,6 +153,11 @@ add_library(vitastor_client SHARED
|
|||
cli_rm_data.cpp
|
||||
cli_rm.cpp
|
||||
cli_rm_osd.cpp
|
||||
cli_pool_cfg.cpp
|
||||
cli_pool_create.cpp
|
||||
cli_pool_ls.cpp
|
||||
cli_pool_modify.cpp
|
||||
cli_pool_rm.cpp
|
||||
)
|
||||
set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
|
||||
target_link_libraries(vitastor_client
|
||||
|
@ -181,10 +185,48 @@ target_link_libraries(vitastor-nbd
|
|||
vitastor_client
|
||||
)
|
||||
|
||||
# libvitastor_kv.so
|
||||
add_library(vitastor_kv SHARED
|
||||
kv_db.cpp
|
||||
kv_db.h
|
||||
)
|
||||
target_link_libraries(vitastor_kv
|
||||
vitastor_client
|
||||
)
|
||||
set_target_properties(vitastor_kv PROPERTIES VERSION ${VERSION} SOVERSION 0)
|
||||
|
||||
# vitastor-kv
|
||||
add_executable(vitastor-kv
|
||||
kv_cli.cpp
|
||||
)
|
||||
target_link_libraries(vitastor-kv
|
||||
vitastor_kv
|
||||
)
|
||||
|
||||
add_executable(vitastor-kv-stress
|
||||
kv_stress.cpp
|
||||
)
|
||||
target_link_libraries(vitastor-kv-stress
|
||||
vitastor_kv
|
||||
)
|
||||
|
||||
# vitastor-nfs
|
||||
add_executable(vitastor-nfs
|
||||
nfs_proxy.cpp
|
||||
nfs_conn.cpp
|
||||
nfs_block.cpp
|
||||
nfs_kv.cpp
|
||||
nfs_kv_create.cpp
|
||||
nfs_kv_getattr.cpp
|
||||
nfs_kv_link.cpp
|
||||
nfs_kv_lookup.cpp
|
||||
nfs_kv_read.cpp
|
||||
nfs_kv_readdir.cpp
|
||||
nfs_kv_remove.cpp
|
||||
nfs_kv_rename.cpp
|
||||
nfs_kv_setattr.cpp
|
||||
nfs_kv_write.cpp
|
||||
nfs_fsstat.cpp
|
||||
nfs_mount.cpp
|
||||
nfs_portmap.cpp
|
||||
sha256.c
|
||||
nfs/xdr_impl.cpp
|
||||
|
@ -194,6 +236,7 @@ add_executable(vitastor-nfs
|
|||
)
|
||||
target_link_libraries(vitastor-nfs
|
||||
vitastor_client
|
||||
vitastor_kv
|
||||
)
|
||||
|
||||
# vitastor-cli
|
||||
|
@ -318,12 +361,12 @@ add_test(NAME test_cluster_client COMMAND test_cluster_client)
|
|||
|
||||
### Install
|
||||
|
||||
install(TARGETS vitastor-osd vitastor-disk vitastor-nbd vitastor-nfs vitastor-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
install(TARGETS vitastor-osd vitastor-disk vitastor-nbd vitastor-nfs vitastor-cli vitastor-kv vitastor-kv-stress RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||
install_symlink(vitastor-disk ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-dump-journal)
|
||||
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm)
|
||||
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita)
|
||||
install(
|
||||
TARGETS vitastor_blk vitastor_client
|
||||
TARGETS vitastor_blk vitastor_client vitastor_kv
|
||||
LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
|
||||
PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}
|
||||
)
|
||||
|
|
|
@ -82,3 +82,8 @@ uint32_t blockstore_t::get_bitmap_granularity()
|
|||
{
|
||||
return impl->get_bitmap_granularity();
|
||||
}
|
||||
|
||||
void blockstore_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
|
||||
{
|
||||
impl->set_no_inode_stats(pool_ids);
|
||||
}
|
||||
|
|
|
@ -216,6 +216,9 @@ public:
|
|||
// Get per-inode space usage statistics
|
||||
std::map<uint64_t, uint64_t> & get_inode_space_stats();
|
||||
|
||||
// Set per-pool no_inode_stats
|
||||
void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
|
||||
|
||||
// Print diagnostics to stdout
|
||||
void dump_diagnostics();
|
||||
|
||||
|
|
|
@ -733,3 +733,86 @@ void blockstore_impl_t::disk_error_abort(const char *op, int retval, int expecte
|
|||
fprintf(stderr, "Disk %s failed: result is %d, expected %d. Can't continue, sorry :-(\n", op, retval, expected);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
void blockstore_impl_t::set_no_inode_stats(const std::vector<uint64_t> & pool_ids)
|
||||
{
|
||||
for (auto & np: no_inode_stats)
|
||||
{
|
||||
np.second = 2;
|
||||
}
|
||||
for (auto pool_id: pool_ids)
|
||||
{
|
||||
if (!no_inode_stats[pool_id])
|
||||
recalc_inode_space_stats(pool_id, false);
|
||||
no_inode_stats[pool_id] = 1;
|
||||
}
|
||||
for (auto np_it = no_inode_stats.begin(); np_it != no_inode_stats.end(); )
|
||||
{
|
||||
if (np_it->second == 2)
|
||||
{
|
||||
recalc_inode_space_stats(np_it->first, true);
|
||||
no_inode_stats.erase(np_it++);
|
||||
}
|
||||
else
|
||||
np_it++;
|
||||
}
|
||||
}
|
||||
|
||||
void blockstore_impl_t::recalc_inode_space_stats(uint64_t pool_id, bool per_inode)
|
||||
{
|
||||
auto sp_begin = inode_space_stats.lower_bound((pool_id << (64-POOL_ID_BITS)));
|
||||
auto sp_end = inode_space_stats.lower_bound(((pool_id+1) << (64-POOL_ID_BITS)));
|
||||
inode_space_stats.erase(sp_begin, sp_end);
|
||||
auto sh_it = clean_db_shards.lower_bound((pool_id << (64-POOL_ID_BITS)));
|
||||
while (sh_it != clean_db_shards.end() &&
|
||||
(sh_it->first >> (64-POOL_ID_BITS)) == pool_id)
|
||||
{
|
||||
for (auto & pair: sh_it->second)
|
||||
{
|
||||
uint64_t space_id = per_inode ? pair.first.inode : (pool_id << (64-POOL_ID_BITS));
|
||||
inode_space_stats[space_id] += dsk.data_block_size;
|
||||
}
|
||||
sh_it++;
|
||||
}
|
||||
object_id last_oid = {};
|
||||
bool last_exists = false;
|
||||
auto dirty_it = dirty_db.lower_bound((obj_ver_id){ .oid = { .inode = (pool_id << (64-POOL_ID_BITS)) } });
|
||||
while (dirty_it != dirty_db.end() && (dirty_it->first.oid.inode >> (64-POOL_ID_BITS)) == pool_id)
|
||||
{
|
||||
if (IS_STABLE(dirty_it->second.state) && (IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state)))
|
||||
{
|
||||
bool exists = false;
|
||||
if (last_oid == dirty_it->first.oid)
|
||||
{
|
||||
exists = last_exists;
|
||||
}
|
||||
else
|
||||
{
|
||||
auto & clean_db = clean_db_shard(dirty_it->first.oid);
|
||||
auto clean_it = clean_db.find(dirty_it->first.oid);
|
||||
exists = clean_it != clean_db.end();
|
||||
}
|
||||
uint64_t space_id = per_inode ? dirty_it->first.oid.inode : (pool_id << (64-POOL_ID_BITS));
|
||||
if (IS_BIG_WRITE(dirty_it->second.state))
|
||||
{
|
||||
if (!exists)
|
||||
inode_space_stats[space_id] += dsk.data_block_size;
|
||||
last_exists = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (exists)
|
||||
{
|
||||
auto & sp = inode_space_stats[space_id];
|
||||
if (sp > dsk.data_block_size)
|
||||
sp -= dsk.data_block_size;
|
||||
else
|
||||
inode_space_stats.erase(space_id);
|
||||
}
|
||||
last_exists = false;
|
||||
}
|
||||
last_oid = dirty_it->first.oid;
|
||||
}
|
||||
dirty_it++;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -272,6 +272,7 @@ class blockstore_impl_t
|
|||
|
||||
std::map<pool_id_t, pool_shard_settings_t> clean_db_settings;
|
||||
std::map<pool_pg_id_t, blockstore_clean_db_t> clean_db_shards;
|
||||
std::map<uint64_t, int> no_inode_stats;
|
||||
uint8_t *clean_bitmaps = NULL;
|
||||
blockstore_dirty_db_t dirty_db;
|
||||
std::vector<blockstore_op_t*> submit_queue;
|
||||
|
@ -318,6 +319,7 @@ class blockstore_impl_t
|
|||
|
||||
blockstore_clean_db_t& clean_db_shard(object_id oid);
|
||||
void reshard_clean_db(pool_id_t pool_id, uint32_t pg_count, uint32_t pg_stripe_size);
|
||||
void recalc_inode_space_stats(uint64_t pool_id, bool per_inode);
|
||||
|
||||
// Journaling
|
||||
void prepare_journal_sector_write(int sector, blockstore_op_t *op);
|
||||
|
@ -428,6 +430,9 @@ public:
|
|||
// Space usage statistics
|
||||
std::map<uint64_t, uint64_t> inode_space_stats;
|
||||
|
||||
// Set per-pool no_inode_stats
|
||||
void set_no_inode_stats(const std::vector<uint64_t> & pool_ids);
|
||||
|
||||
// Print diagnostics to stdout
|
||||
void dump_diagnostics();
|
||||
|
||||
|
|
|
@ -32,7 +32,7 @@ void blockstore_init_meta::handle_event(ring_data_t *data, int buf_num)
|
|||
if (data->res < 0)
|
||||
{
|
||||
throw std::runtime_error(
|
||||
std::string("read metadata failed at offset ") + std::to_string(bufs[buf_num].offset) +
|
||||
std::string("read metadata failed at offset ") + std::to_string(buf_num >= 0 ? bufs[buf_num].offset : last_read_offset) +
|
||||
std::string(": ") + strerror(-data->res)
|
||||
);
|
||||
}
|
||||
|
@ -63,6 +63,7 @@ int blockstore_init_meta::loop()
|
|||
throw std::runtime_error("Failed to allocate metadata read buffer");
|
||||
// Read superblock
|
||||
GET_SQE();
|
||||
last_read_offset = 0;
|
||||
data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
|
||||
data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
|
||||
my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
|
||||
|
@ -100,6 +101,7 @@ resume_1:
|
|||
{
|
||||
printf("Initializing metadata area\n");
|
||||
GET_SQE();
|
||||
last_read_offset = 0;
|
||||
data->iov = (struct iovec){ metadata_buffer, (size_t)bs->dsk.meta_block_size };
|
||||
data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
|
||||
my_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset);
|
||||
|
@ -236,6 +238,7 @@ resume_2:
|
|||
data->iov = { bufs[i].buf, (size_t)bufs[i].size };
|
||||
data->callback = [this, i](ring_data_t *data) { handle_event(data, i); };
|
||||
my_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bufs[i].offset);
|
||||
bs->ringloop->submit();
|
||||
bufs[i].state = INIT_META_WRITING;
|
||||
submitted++;
|
||||
}
|
||||
|
@ -259,9 +262,11 @@ resume_2:
|
|||
next_offset = entries_to_zero[i]/entries_per_block;
|
||||
for (j = i; j < entries_to_zero.size() && entries_to_zero[j]/entries_per_block == next_offset; j++) {}
|
||||
GET_SQE();
|
||||
last_read_offset = (1+next_offset)*bs->dsk.meta_block_size;
|
||||
data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
|
||||
data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
|
||||
my_uring_prep_readv(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
|
||||
bs->ringloop->submit();
|
||||
submitted++;
|
||||
resume_5:
|
||||
if (submitted > 0)
|
||||
|
@ -278,6 +283,7 @@ resume_5:
|
|||
data->iov = { metadata_buffer, (size_t)bs->dsk.meta_block_size };
|
||||
data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
|
||||
my_uring_prep_writev(sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + (1+next_offset)*bs->dsk.meta_block_size);
|
||||
bs->ringloop->submit();
|
||||
submitted++;
|
||||
resume_6:
|
||||
if (submitted > 0)
|
||||
|
@ -299,6 +305,7 @@ resume_6:
|
|||
{
|
||||
GET_SQE();
|
||||
my_uring_prep_fsync(sqe, bs->dsk.meta_fd, IORING_FSYNC_DATASYNC);
|
||||
last_read_offset = 0;
|
||||
data->iov = { 0 };
|
||||
data->callback = [this](ring_data_t *data) { handle_event(data, -1); };
|
||||
submitted++;
|
||||
|
|
|
@ -23,6 +23,7 @@ class blockstore_init_meta
|
|||
struct ring_data_t *data;
|
||||
uint64_t md_offset = 0;
|
||||
uint64_t next_offset = 0;
|
||||
uint64_t last_read_offset = 0;
|
||||
uint64_t entries_loaded = 0;
|
||||
unsigned entries_per_block = 0;
|
||||
int i = 0, j = 0;
|
||||
|
|
|
@ -487,18 +487,24 @@ void blockstore_impl_t::mark_stable(obj_ver_id v, bool forget_dirty)
|
|||
}
|
||||
if (!exists)
|
||||
{
|
||||
inode_space_stats[dirty_it->first.oid.inode] += dsk.data_block_size;
|
||||
uint64_t space_id = dirty_it->first.oid.inode;
|
||||
if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
|
||||
space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
|
||||
inode_space_stats[space_id] += dsk.data_block_size;
|
||||
used_blocks++;
|
||||
}
|
||||
big_to_flush++;
|
||||
}
|
||||
else if (IS_DELETE(dirty_it->second.state))
|
||||
{
|
||||
auto & sp = inode_space_stats[dirty_it->first.oid.inode];
|
||||
uint64_t space_id = dirty_it->first.oid.inode;
|
||||
if (no_inode_stats[dirty_it->first.oid.inode >> (64-POOL_ID_BITS)])
|
||||
space_id = space_id & ~(((uint64_t)1 << (64-POOL_ID_BITS)) - 1);
|
||||
auto & sp = inode_space_stats[space_id];
|
||||
if (sp > dsk.data_block_size)
|
||||
sp -= dsk.data_block_size;
|
||||
else
|
||||
inode_space_stats.erase(dirty_it->first.oid.inode);
|
||||
inode_space_stats.erase(space_id);
|
||||
used_blocks--;
|
||||
big_to_flush++;
|
||||
}
|
||||
|
|
120
src/cli.cpp
120
src/cli.cpp
|
@ -46,18 +46,21 @@ static const char* help_text =
|
|||
"vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>\n"
|
||||
" Create a snapshot of image <name>. May be used live if only a single writer is active.\n"
|
||||
"\n"
|
||||
"vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force]\n"
|
||||
"vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force] [--down-ok]\n"
|
||||
" Rename, resize image or change its readonly status. Images with children can't be made read-write.\n"
|
||||
" If the new size is smaller than the old size, extra data will be purged.\n"
|
||||
" You should resize file system in the image, if present, before shrinking it.\n"
|
||||
" -f|--force Proceed with shrinking or setting readwrite flag even if the image has children.\n"
|
||||
" --down-ok Proceed with shrinking even if some data will be left on unavailable OSDs.\n"
|
||||
"\n"
|
||||
"vitastor-cli rm <from> [<to>] [--writers-stopped]\n"
|
||||
"vitastor-cli rm <from> [<to>] [--writers-stopped] [--down-ok]\n"
|
||||
" Remove <from> or all layers between <from> and <to> (<to> must be a child of <from>),\n"
|
||||
" rebasing all their children accordingly. --writers-stopped allows merging to be a bit\n"
|
||||
" more effective in case of a single 'slim' read-write child and 'fat' removed parent:\n"
|
||||
" the child is merged into parent and parent is renamed to child in that case.\n"
|
||||
" In other cases parent layers are always merged into children.\n"
|
||||
" Other options:\n"
|
||||
" --down-ok Continue deletion/merging even if some data will be left on unavailable OSDs.\n"
|
||||
"\n"
|
||||
"vitastor-cli flatten <layer>\n"
|
||||
" Flatten a layer, i.e. merge data and detach it from parents.\n"
|
||||
|
@ -113,6 +116,55 @@ static const char* help_text =
|
|||
" With --dry-run only checks if deletion is possible without data loss and\n"
|
||||
" redundancy degradation.\n"
|
||||
"\n"
|
||||
"vitastor-cli create-pool|pool-create <name> (-s <pg_size>|--ec <N>+<K>) -n <pg_count> [OPTIONS]\n"
|
||||
" Create a pool. Required parameters:\n"
|
||||
" -s|--pg_size R Number of replicas for replicated pools\n"
|
||||
" --ec N+K Number of data (N) and parity (K) chunks for erasure-coded pools\n"
|
||||
" -n|--pg_count N PG count for the new pool (start with 10*<OSD count>/pg_size rounded to a power of 2)\n"
|
||||
" Optional parameters:\n"
|
||||
" --pg_minsize <number> R or N+K minus number of failures to tolerate without downtime\n"
|
||||
" --failure_domain host Failure domain: host, osd or a level from placement_levels. Default: host\n"
|
||||
" --root_node <node> Put pool only on child OSDs of this placement tree node\n"
|
||||
" --osd_tags <tag>[,<tag>]... Put pool only on OSDs tagged with all specified tags\n"
|
||||
" --block_size 128k Put pool only on OSDs with this data block size\n"
|
||||
" --bitmap_granularity 4k Put pool only on OSDs with this logical sector size\n"
|
||||
" --immediate_commit none Put pool only on OSDs with this or larger immediate_commit (none < small < all)\n"
|
||||
" --primary_affinity_tags tags Prefer to put primary copies on OSDs with all specified tags\n"
|
||||
" --scrub_interval <time> Enable regular scrubbing for this pool. Format: number + unit s/m/h/d/M/y\n"
|
||||
" --used_for_fs <name> Mark pool as used for VitastorFS with metadata in image <name>\n"
|
||||
" --pg_stripe_size <number> Increase object grouping stripe\n"
|
||||
" --max_osd_combinations 10000 Maximum number of random combinations for LP solver input\n"
|
||||
" --wait Wait for the new pool to come online\n"
|
||||
" -f|--force Do not check that cluster has enough OSDs to create the pool\n"
|
||||
" Examples:\n"
|
||||
" vitastor-cli create-pool test_x4 -s 4 -n 32\n"
|
||||
" vitastor-cli create-pool test_ec42 --ec 4+2 -n 32\n"
|
||||
"\n"
|
||||
"vitastor-cli modify-pool|pool-modify <id|name> [--name <new_name>] [PARAMETERS...]\n"
|
||||
" Modify an existing pool. Modifiable parameters:\n"
|
||||
" [-s|--pg_size <number>] [--pg_minsize <number>] [-n|--pg_count <count>]\n"
|
||||
" [--failure_domain <level>] [--root_node <node>] [--osd_tags <tags>] [--used_for_fs <name>]\n"
|
||||
" [--max_osd_combinations <number>] [--primary_affinity_tags <tags>] [--scrub_interval <time>]\n"
|
||||
" Non-modifiable parameters (changing them WILL lead to data loss):\n"
|
||||
" [--block_size <size>] [--bitmap_granularity <size>]\n"
|
||||
" [--immediate_commit <all|small|none>] [--pg_stripe_size <size>]\n"
|
||||
" These, however, can still be modified with -f|--force.\n"
|
||||
" See create-pool for parameter descriptions.\n"
|
||||
" Examples:\n"
|
||||
" vitastor-cli modify-pool pool_A --name pool_B\n"
|
||||
" vitastor-cli modify-pool 2 --pg_size 4 -n 128\n"
|
||||
"\n"
|
||||
"vitastor-cli rm-pool|pool-rm [--force] <id|name>\n"
|
||||
" Remove a pool. Refuses to remove pools with images without --force.\n"
|
||||
"\n"
|
||||
"vitastor-cli ls-pools|pool-ls|ls-pool|pools [-l] [--detail] [--sort FIELD] [-r] [-n N] [--stats] [<glob> ...]\n"
|
||||
" List pools (only matching <glob> patterns if passed).\n"
|
||||
" -l|--long Also report I/O statistics\n"
|
||||
" --detail Use list format (not table), show all details\n"
|
||||
" --sort FIELD Sort by specified field (see fields in --json output)\n"
|
||||
" -r|--reverse Sort in descending order\n"
|
||||
" -n|--count N Only list first N items\n"
|
||||
"\n"
|
||||
"Use vitastor-cli --help <command> for command details or vitastor-cli --help --all for all details.\n"
|
||||
"\n"
|
||||
"GLOBAL OPTIONS:\n"
|
||||
|
@ -122,7 +174,7 @@ static const char* help_text =
|
|||
" --parallel_osds M Work with M osds in parallel when possible (default 4)\n"
|
||||
" --progress 1|0 Report progress (default 1)\n"
|
||||
" --cas 1|0 Use CAS writes for flatten, merge, rm (default is decide automatically)\n"
|
||||
" --no-color Disable colored output\n"
|
||||
" --color 1|0 Enable/disable colored output and CR symbols (default 1 if stdout is a terminal)\n"
|
||||
" --json JSON output\n"
|
||||
;
|
||||
|
||||
|
@ -133,6 +185,7 @@ static json11::Json::object parse_args(int narg, const char *args[])
|
|||
cfg["progress"] = "1";
|
||||
for (int i = 1; i < narg; i++)
|
||||
{
|
||||
bool argHasValue = (!(i == narg-1) && (args[i+1][0] != '-'));
|
||||
if (args[i][0] == '-' && args[i][1] == 'h' && args[i][2] == 0)
|
||||
{
|
||||
cfg["help"] = "1";
|
||||
|
@ -143,15 +196,15 @@ static json11::Json::object parse_args(int narg, const char *args[])
|
|||
}
|
||||
else if (args[i][0] == '-' && args[i][1] == 'n' && args[i][2] == 0)
|
||||
{
|
||||
cfg["count"] = args[++i];
|
||||
cfg["count"] = argHasValue ? args[++i] : "";
|
||||
}
|
||||
else if (args[i][0] == '-' && args[i][1] == 'p' && args[i][2] == 0)
|
||||
{
|
||||
cfg["pool"] = args[++i];
|
||||
cfg["pool"] = argHasValue ? args[++i] : "";
|
||||
}
|
||||
else if (args[i][0] == '-' && args[i][1] == 's' && args[i][2] == 0)
|
||||
{
|
||||
cfg["size"] = args[++i];
|
||||
cfg["size"] = argHasValue ? args[++i] : "";
|
||||
}
|
||||
else if (args[i][0] == '-' && args[i][1] == 'r' && args[i][2] == 0)
|
||||
{
|
||||
|
@ -164,17 +217,24 @@ static json11::Json::object parse_args(int narg, const char *args[])
|
|||
else if (args[i][0] == '-' && args[i][1] == '-')
|
||||
{
|
||||
const char *opt = args[i]+2;
|
||||
cfg[opt] = i == narg-1 || !strcmp(opt, "json") ||
|
||||
if (!strcmp(opt, "json") || !strcmp(opt, "wait") ||
|
||||
!strcmp(opt, "wait-list") || !strcmp(opt, "wait_list") ||
|
||||
!strcmp(opt, "long") || !strcmp(opt, "del") ||
|
||||
!strcmp(opt, "long") || !strcmp(opt, "detail") || !strcmp(opt, "del") ||
|
||||
!strcmp(opt, "no-color") || !strcmp(opt, "no_color") ||
|
||||
!strcmp(opt, "readonly") || !strcmp(opt, "readwrite") ||
|
||||
!strcmp(opt, "force") || !strcmp(opt, "reverse") ||
|
||||
!strcmp(opt, "allow-data-loss") || !strcmp(opt, "allow_data_loss") ||
|
||||
!strcmp(opt, "down-ok") || !strcmp(opt, "down_ok") ||
|
||||
!strcmp(opt, "dry-run") || !strcmp(opt, "dry_run") ||
|
||||
!strcmp(opt, "help") || !strcmp(opt, "all") ||
|
||||
(!strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped")) && strcmp("1", args[i+1]) != 0
|
||||
? "1" : args[++i];
|
||||
!strcmp(opt, "writers-stopped") || !strcmp(opt, "writers_stopped"))
|
||||
{
|
||||
cfg[opt] = "1";
|
||||
}
|
||||
else
|
||||
{
|
||||
cfg[opt] = argHasValue ? args[++i] : "";
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
|
@ -217,7 +277,7 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
|
|||
else if (cmd[0] == "df")
|
||||
{
|
||||
// Show pool space stats
|
||||
action_cb = p->start_df(cfg);
|
||||
action_cb = p->start_pool_ls(cfg);
|
||||
}
|
||||
else if (cmd[0] == "ls")
|
||||
{
|
||||
|
@ -324,6 +384,44 @@ static int run(cli_tool_t *p, json11::Json::object cfg)
|
|||
// Allocate a new OSD number
|
||||
action_cb = p->start_alloc_osd(cfg);
|
||||
}
|
||||
else if (cmd[0] == "create-pool" || cmd[0] == "pool-create")
|
||||
{
|
||||
// Create a new pool
|
||||
if (cmd.size() > 1 && cfg["name"].is_null())
|
||||
{
|
||||
cfg["name"] = cmd[1];
|
||||
}
|
||||
action_cb = p->start_pool_create(cfg);
|
||||
}
|
||||
else if (cmd[0] == "modify-pool" || cmd[0] == "pool-modify")
|
||||
{
|
||||
// Modify existing pool
|
||||
if (cmd.size() > 1)
|
||||
{
|
||||
cfg["old_name"] = cmd[1];
|
||||
}
|
||||
action_cb = p->start_pool_modify(cfg);
|
||||
}
|
||||
else if (cmd[0] == "rm-pool" || cmd[0] == "pool-rm")
|
||||
{
|
||||
// Remove existing pool
|
||||
if (cmd.size() > 1)
|
||||
{
|
||||
cfg["pool"] = cmd[1];
|
||||
}
|
||||
action_cb = p->start_pool_rm(cfg);
|
||||
}
|
||||
else if (cmd[0] == "ls-pool" || cmd[0] == "pool-ls" || cmd[0] == "ls-pools" || cmd[0] == "pools")
|
||||
{
|
||||
// Show pool list
|
||||
cfg["show_recovery"] = 1;
|
||||
if (cmd.size() > 1)
|
||||
{
|
||||
cmd.erase(cmd.begin(), cmd.begin()+1);
|
||||
cfg["names"] = cmd;
|
||||
}
|
||||
action_cb = p->start_pool_ls(cfg);
|
||||
}
|
||||
else
|
||||
{
|
||||
result = { .err = EINVAL, .text = "unknown command: "+cmd[0].string_value() };
|
||||
|
|
11
src/cli.h
11
src/cli.h
|
@ -46,6 +46,7 @@ public:
|
|||
json11::Json etcd_result;
|
||||
|
||||
void parse_config(json11::Json::object & cfg);
|
||||
json11::Json parse_tags(std::string tags);
|
||||
|
||||
void change_parent(inode_t cur, inode_t new_parent, cli_result_t *result);
|
||||
inode_config_t* get_inode_cfg(const std::string & name);
|
||||
|
@ -58,7 +59,6 @@ public:
|
|||
std::function<bool(cli_result_t &)> start_status(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_describe(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_fix(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_df(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_ls(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_create(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_modify(json11::Json);
|
||||
|
@ -68,6 +68,10 @@ public:
|
|||
std::function<bool(cli_result_t &)> start_rm(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_rm_osd(json11::Json cfg);
|
||||
std::function<bool(cli_result_t &)> start_alloc_osd(json11::Json cfg);
|
||||
std::function<bool(cli_result_t &)> start_pool_create(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_pool_modify(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_pool_rm(json11::Json);
|
||||
std::function<bool(cli_result_t &)> start_pool_ls(json11::Json);
|
||||
|
||||
// Should be called like loop_and_wait(start_status(), <completion callback>)
|
||||
void loop_and_wait(std::function<bool(cli_result_t &)> loop_cb, std::function<void(const cli_result_t &)> complete_cb);
|
||||
|
@ -77,8 +81,13 @@ public:
|
|||
|
||||
std::string print_table(json11::Json items, json11::Json header, bool use_esc);
|
||||
|
||||
size_t print_detail_title_len(json11::Json item, std::vector<std::pair<std::string, std::string>> names, size_t prev_len);
|
||||
std::string print_detail(json11::Json item, std::vector<std::pair<std::string, std::string>> names, size_t title_len, bool use_esc);
|
||||
|
||||
std::string format_lat(uint64_t lat);
|
||||
|
||||
std::string format_q(double depth);
|
||||
|
||||
bool stupid_glob(const std::string str, const std::string glob);
|
||||
|
||||
std::string implode(const std::string & sep, json11::Json array);
|
||||
|
|
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <unistd.h>
|
||||
#include "str_util.h"
|
||||
#include "cluster_client.h"
|
||||
#include "cli.h"
|
||||
|
@ -113,7 +114,12 @@ void cli_tool_t::parse_config(json11::Json::object & cfg)
|
|||
else
|
||||
kv_it++;
|
||||
}
|
||||
if (cfg.find("no_color") != cfg.end())
|
||||
color = !cfg["no_color"].bool_value();
|
||||
else if (cfg.find("color") != cfg.end())
|
||||
color = cfg["color"].bool_value();
|
||||
else
|
||||
color = isatty(1);
|
||||
json_output = cfg["json"].bool_value();
|
||||
iodepth = cfg["iodepth"].uint64_value();
|
||||
if (!iodepth)
|
||||
|
@ -147,6 +153,7 @@ void cli_tool_t::loop_and_wait(std::function<bool(cli_result_t &)> loop_cb, std:
|
|||
ringloop->unregister_consumer(&looper->consumer);
|
||||
looper->loop_cb = NULL;
|
||||
looper->complete_cb(looper->result);
|
||||
ringloop->submit();
|
||||
delete looper;
|
||||
return;
|
||||
}
|
||||
|
|
|
@ -27,6 +27,7 @@ struct image_creator_t
|
|||
std::string image_name, new_snap, new_parent;
|
||||
json11::Json new_meta;
|
||||
uint64_t size;
|
||||
bool force = false;
|
||||
bool force_size = false;
|
||||
|
||||
pool_id_t old_pool_id = 0;
|
||||
|
@ -45,6 +46,7 @@ struct image_creator_t
|
|||
|
||||
void loop()
|
||||
{
|
||||
auto & pools = parent->cli->st_cli.pool_config;
|
||||
if (state >= 1)
|
||||
goto resume_1;
|
||||
if (image_name == "")
|
||||
|
@ -62,7 +64,6 @@ struct image_creator_t
|
|||
}
|
||||
if (new_pool_id)
|
||||
{
|
||||
auto & pools = parent->cli->st_cli.pool_config;
|
||||
if (pools.find(new_pool_id) == pools.end())
|
||||
{
|
||||
result = (cli_result_t){ .err = ENOENT, .text = "Pool "+std::to_string(new_pool_id)+" does not exist" };
|
||||
|
@ -72,7 +73,7 @@ struct image_creator_t
|
|||
}
|
||||
else if (new_pool_name != "")
|
||||
{
|
||||
for (auto & ic: parent->cli->st_cli.pool_config)
|
||||
for (auto & ic: pools)
|
||||
{
|
||||
if (ic.second.name == new_pool_name)
|
||||
{
|
||||
|
@ -87,10 +88,20 @@ struct image_creator_t
|
|||
return;
|
||||
}
|
||||
}
|
||||
else if (parent->cli->st_cli.pool_config.size() == 1)
|
||||
else if (pools.size() == 1)
|
||||
{
|
||||
auto it = parent->cli->st_cli.pool_config.begin();
|
||||
new_pool_id = it->first;
|
||||
new_pool_id = pools.begin()->first;
|
||||
}
|
||||
if (new_pool_id && !pools.at(new_pool_id).used_for_fs.empty() && !force)
|
||||
{
|
||||
result = (cli_result_t){
|
||||
.err = EINVAL,
|
||||
.text = "Pool "+pools.at(new_pool_id).name+
|
||||
" is used for VitastorFS "+pools.at(new_pool_id).used_for_fs+
|
||||
". Use --force if you really know what you are doing",
|
||||
};
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
state = 1;
|
||||
resume_1:
|
||||
|
@ -183,7 +194,16 @@ resume_3:
|
|||
// Save into inode_config for library users to be able to take it from there immediately
|
||||
new_cfg.mod_revision = parent->etcd_result["responses"][0]["response_put"]["header"]["revision"].uint64_value();
|
||||
parent->cli->st_cli.insert_inode_config(new_cfg);
|
||||
result = (cli_result_t){ .err = 0, .text = "Image "+image_name+" created" };
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Image "+image_name+" created",
|
||||
.data = json11::Json::object {
|
||||
{ "name", image_name },
|
||||
{ "pool", new_pool_name },
|
||||
{ "parent", new_parent },
|
||||
{ "size", size },
|
||||
}
|
||||
};
|
||||
state = 100;
|
||||
}
|
||||
|
||||
|
@ -251,7 +271,16 @@ resume_4:
|
|||
// Save into inode_config for library users to be able to take it from there immediately
|
||||
new_cfg.mod_revision = parent->etcd_result["responses"][0]["response_put"]["header"]["revision"].uint64_value();
|
||||
parent->cli->st_cli.insert_inode_config(new_cfg);
|
||||
result = (cli_result_t){ .err = 0, .text = "Snapshot "+image_name+"@"+new_snap+" created" };
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Snapshot "+image_name+"@"+new_snap+" created",
|
||||
.data = json11::Json::object {
|
||||
{ "name", image_name+"@"+new_snap },
|
||||
{ "pool", (uint64_t)new_pool_id },
|
||||
{ "parent", new_parent },
|
||||
{ "size", size },
|
||||
}
|
||||
};
|
||||
state = 100;
|
||||
}
|
||||
|
||||
|
@ -514,6 +543,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_create(json11::Json cfg)
|
|||
image_creator->image_name = cfg["image"].string_value();
|
||||
image_creator->new_pool_id = cfg["pool"].uint64_value();
|
||||
image_creator->new_pool_name = cfg["pool"].string_value();
|
||||
image_creator->force = cfg["force"].bool_value();
|
||||
image_creator->force_size = cfg["force_size"].bool_value();
|
||||
if (cfg["image_meta"].is_object())
|
||||
{
|
||||
|
|
243
src/cli_df.cpp
243
src/cli_df.cpp
|
@ -1,243 +0,0 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "cli.h"
|
||||
#include "cluster_client.h"
|
||||
#include "str_util.h"
|
||||
|
||||
// List pools with space statistics
|
||||
struct pool_lister_t
|
||||
{
|
||||
cli_tool_t *parent;
|
||||
|
||||
int state = 0;
|
||||
json11::Json space_info;
|
||||
cli_result_t result;
|
||||
std::map<pool_id_t, json11::Json::object> pool_stats;
|
||||
|
||||
bool is_done()
|
||||
{
|
||||
return state == 100;
|
||||
}
|
||||
|
||||
void get_stats()
|
||||
{
|
||||
if (state == 1)
|
||||
goto resume_1;
|
||||
// Space statistics - pool/stats/<pool>
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pool/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pool/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/osd/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/osd/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 1;
|
||||
resume_1:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
space_info = parent->etcd_result;
|
||||
std::map<pool_id_t, uint64_t> osd_free;
|
||||
for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// pool ID
|
||||
pool_id_t pool_id;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
|
||||
if (scanned != 1 || !pool_id || pool_id >= POOL_ID_MAX)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
// pool/stats/<N>
|
||||
pool_stats[pool_id] = kv.value.object_items();
|
||||
}
|
||||
for (auto & kv_item: space_info["responses"][1]["response_range"]["kvs"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// osd ID
|
||||
osd_num_t osd_num;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/osd/stats/%ju%c", &osd_num, &null_byte);
|
||||
if (scanned != 1 || !osd_num || osd_num >= POOL_ID_MAX)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
// osd/stats/<N>::free
|
||||
osd_free[osd_num] = kv.value["free"].uint64_value();
|
||||
}
|
||||
// Calculate max_avail for each pool
|
||||
for (auto & pp: parent->cli->st_cli.pool_config)
|
||||
{
|
||||
auto & pool_cfg = pp.second;
|
||||
uint64_t pool_avail = UINT64_MAX;
|
||||
std::map<osd_num_t, uint64_t> pg_per_osd;
|
||||
for (auto & pgp: pool_cfg.pg_config)
|
||||
{
|
||||
for (auto pg_osd: pgp.second.target_set)
|
||||
{
|
||||
if (pg_osd != 0)
|
||||
{
|
||||
pg_per_osd[pg_osd]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto pg_per_pair: pg_per_osd)
|
||||
{
|
||||
uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
|
||||
if (pool_avail > pg_free)
|
||||
{
|
||||
pool_avail = pg_free;
|
||||
}
|
||||
}
|
||||
if (pool_avail == UINT64_MAX)
|
||||
{
|
||||
pool_avail = 0;
|
||||
}
|
||||
if (pool_cfg.scheme != POOL_SCHEME_REPLICATED)
|
||||
{
|
||||
pool_avail *= (pool_cfg.pg_size - pool_cfg.parity_chunks);
|
||||
}
|
||||
pool_stats[pool_cfg.id] = json11::Json::object {
|
||||
{ "id", (uint64_t)pool_cfg.id },
|
||||
{ "name", pool_cfg.name },
|
||||
{ "pg_count", pool_cfg.pg_count },
|
||||
{ "real_pg_count", pool_cfg.real_pg_count },
|
||||
{ "scheme", pool_cfg.scheme == POOL_SCHEME_REPLICATED ? "replicated" : "ec" },
|
||||
{ "scheme_name", pool_cfg.scheme == POOL_SCHEME_REPLICATED
|
||||
? std::to_string(pool_cfg.pg_size)+"/"+std::to_string(pool_cfg.pg_minsize)
|
||||
: "EC "+std::to_string(pool_cfg.pg_size-pool_cfg.parity_chunks)+"+"+std::to_string(pool_cfg.parity_chunks) },
|
||||
{ "used_raw", (uint64_t)(pool_stats[pool_cfg.id]["used_raw_tb"].number_value() * ((uint64_t)1<<40)) },
|
||||
{ "total_raw", (uint64_t)(pool_stats[pool_cfg.id]["total_raw_tb"].number_value() * ((uint64_t)1<<40)) },
|
||||
{ "max_available", pool_avail },
|
||||
{ "raw_to_usable", pool_stats[pool_cfg.id]["raw_to_usable"].number_value() },
|
||||
{ "space_efficiency", pool_stats[pool_cfg.id]["space_efficiency"].number_value() },
|
||||
{ "pg_real_size", pool_stats[pool_cfg.id]["pg_real_size"].uint64_value() },
|
||||
{ "failure_domain", pool_cfg.failure_domain },
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
json11::Json::array to_list()
|
||||
{
|
||||
json11::Json::array list;
|
||||
for (auto & kv: pool_stats)
|
||||
{
|
||||
list.push_back(kv.second);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
void loop()
|
||||
{
|
||||
get_stats();
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (state == 100)
|
||||
return;
|
||||
if (parent->json_output)
|
||||
{
|
||||
// JSON output
|
||||
result.data = to_list();
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
// Table output: name, scheme_name, pg_count, total, used, max_avail, used%, efficiency
|
||||
json11::Json::array cols;
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "name" },
|
||||
{ "title", "NAME" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "scheme_name" },
|
||||
{ "title", "SCHEME" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "pg_count_fmt" },
|
||||
{ "title", "PGS" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "total_fmt" },
|
||||
{ "title", "TOTAL" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "used_fmt" },
|
||||
{ "title", "USED" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "max_avail_fmt" },
|
||||
{ "title", "AVAILABLE" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "used_pct" },
|
||||
{ "title", "USED%" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "eff_fmt" },
|
||||
{ "title", "EFFICIENCY" },
|
||||
});
|
||||
json11::Json::array list;
|
||||
for (auto & kv: pool_stats)
|
||||
{
|
||||
double raw_to = kv.second["raw_to_usable"].number_value();
|
||||
if (raw_to < 0.000001 && raw_to > -0.000001)
|
||||
raw_to = 1;
|
||||
kv.second["pg_count_fmt"] = kv.second["real_pg_count"] == kv.second["pg_count"]
|
||||
? kv.second["real_pg_count"].as_string()
|
||||
: kv.second["real_pg_count"].as_string()+"->"+kv.second["pg_count"].as_string();
|
||||
kv.second["total_fmt"] = format_size(kv.second["total_raw"].uint64_value() / raw_to);
|
||||
kv.second["used_fmt"] = format_size(kv.second["used_raw"].uint64_value() / raw_to);
|
||||
kv.second["max_avail_fmt"] = format_size(kv.second["max_available"].uint64_value());
|
||||
kv.second["used_pct"] = format_q(kv.second["total_raw"].uint64_value()
|
||||
? (100 - 100*kv.second["max_available"].uint64_value() *
|
||||
kv.second["raw_to_usable"].number_value() / kv.second["total_raw"].uint64_value())
|
||||
: 100)+"%";
|
||||
kv.second["eff_fmt"] = format_q(kv.second["space_efficiency"].number_value()*100)+"%";
|
||||
}
|
||||
result.data = to_list();
|
||||
result.text = print_table(result.data, cols, parent->color);
|
||||
state = 100;
|
||||
}
|
||||
};
|
||||
|
||||
std::function<bool(cli_result_t &)> cli_tool_t::start_df(json11::Json cfg)
|
||||
{
|
||||
auto lister = new pool_lister_t();
|
||||
lister->parent = this;
|
||||
return [lister](cli_result_t & result)
|
||||
{
|
||||
lister->loop();
|
||||
if (lister->is_done())
|
||||
{
|
||||
result = lister->result;
|
||||
delete lister;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
|
@ -275,7 +275,9 @@ struct snap_merger_t
|
|||
processed++;
|
||||
if (parent->progress && !(processed % 128))
|
||||
{
|
||||
printf("\rFiltering target blocks: %ju/%ju", processed, to_process);
|
||||
fprintf(stderr, parent->color
|
||||
? "\rFiltering target blocks: %ju/%ju"
|
||||
: "Filtering target blocks: %ju/%ju\n", processed, to_process);
|
||||
}
|
||||
}
|
||||
if (in_flight > 0 || oit != merge_offsets.end())
|
||||
|
@ -285,7 +287,9 @@ struct snap_merger_t
|
|||
}
|
||||
if (parent->progress)
|
||||
{
|
||||
printf("\r%ju full blocks of target filtered out\n", to_process-merge_offsets.size());
|
||||
fprintf(stderr, parent->color
|
||||
? "\r%ju full blocks of target filtered out\n"
|
||||
: "%ju full blocks of target filtered out\n", to_process-merge_offsets.size());
|
||||
}
|
||||
}
|
||||
state = 3;
|
||||
|
@ -320,7 +324,9 @@ struct snap_merger_t
|
|||
processed++;
|
||||
if (parent->progress && !(processed % 128))
|
||||
{
|
||||
printf("\rOverwriting blocks: %ju/%ju", processed, to_process);
|
||||
fprintf(stderr, parent->color
|
||||
? "\rOverwriting blocks: %ju/%ju"
|
||||
: "Overwriting blocks: %ju/%ju\n", processed, to_process);
|
||||
}
|
||||
}
|
||||
if (in_flight == 0 && rwo_error.size())
|
||||
|
@ -339,10 +345,16 @@ struct snap_merger_t
|
|||
}
|
||||
if (parent->progress)
|
||||
{
|
||||
printf("\rOverwriting blocks: %ju/%ju\n", to_process, to_process);
|
||||
fprintf(stderr, parent->color
|
||||
? "\rOverwriting blocks: %ju/%ju\n"
|
||||
: "Overwriting blocks: %ju/%ju\n", to_process, to_process);
|
||||
}
|
||||
// Done
|
||||
result = (cli_result_t){ .text = "Done, layers from "+from_name+" to "+to_name+" merged into "+target_name };
|
||||
result = (cli_result_t){ .text = "Done, layers from "+from_name+" to "+to_name+" merged into "+target_name, .data = json11::Json::object {
|
||||
{ "from", from_name },
|
||||
{ "to", to_name },
|
||||
{ "into", target_name },
|
||||
}};
|
||||
state = 100;
|
||||
resume_100:
|
||||
return;
|
||||
|
|
|
@ -15,6 +15,7 @@ struct image_changer_t
|
|||
uint64_t new_size = 0;
|
||||
bool force_size = false, inc_size = false;
|
||||
bool set_readonly = false, set_readwrite = false, force = false;
|
||||
bool down_ok = false;
|
||||
// interval between fsyncs
|
||||
int fsync_interval = 128;
|
||||
|
||||
|
@ -84,7 +85,10 @@ struct image_changer_t
|
|||
(!new_size && !force_size || cfg.size == new_size || cfg.size >= new_size && inc_size) &&
|
||||
(new_name == "" || new_name == image_name))
|
||||
{
|
||||
result = (cli_result_t){ .text = "No change" };
|
||||
result = (cli_result_t){ .err = 0, .text = "No change", .data = json11::Json::object {
|
||||
{ "error_code", 0 },
|
||||
{ "error_text", "No change" },
|
||||
}};
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
@ -105,6 +109,7 @@ struct image_changer_t
|
|||
{ "pool", (uint64_t)INODE_POOL(inode_num) },
|
||||
{ "fsync-interval", fsync_interval },
|
||||
{ "min-offset", ((new_size+4095)/4096)*4096 },
|
||||
{ "down-ok", down_ok },
|
||||
});
|
||||
resume_1:
|
||||
while (!cb(result))
|
||||
|
@ -220,7 +225,16 @@ resume_2:
|
|||
parent->cli->st_cli.inode_by_name.erase(image_name);
|
||||
}
|
||||
parent->cli->st_cli.insert_inode_config(cfg);
|
||||
result = (cli_result_t){ .err = 0, .text = "Image "+image_name+" modified" };
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Image "+image_name+" modified",
|
||||
.data = json11::Json::object {
|
||||
{ "name", image_name },
|
||||
{ "inode", INODE_NO_POOL(inode_num) },
|
||||
{ "pool", (uint64_t)INODE_POOL(inode_num) },
|
||||
{ "size", new_size },
|
||||
}
|
||||
};
|
||||
state = 100;
|
||||
}
|
||||
};
|
||||
|
@ -240,6 +254,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_modify(json11::Json cfg)
|
|||
changer->fsync_interval = cfg["fsync_interval"].uint64_value();
|
||||
if (!changer->fsync_interval)
|
||||
changer->fsync_interval = 128;
|
||||
changer->down_ok = cfg["down_ok"].bool_value();
|
||||
// FIXME Check that the image doesn't have children when shrinking
|
||||
return [changer](cli_result_t & result)
|
||||
{
|
||||
|
|
|
@ -0,0 +1,270 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2024
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include "cli_pool_cfg.h"
|
||||
#include "etcd_state_client.h"
|
||||
#include "str_util.h"
|
||||
|
||||
std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json old_cfg,
|
||||
uint64_t global_block_size, uint64_t global_bitmap_granularity, bool force)
|
||||
{
|
||||
// short option names
|
||||
if (new_cfg.find("count") != new_cfg.end())
|
||||
{
|
||||
new_cfg["pg_count"] = new_cfg["count"];
|
||||
new_cfg.erase("count");
|
||||
}
|
||||
if (new_cfg.find("size") != new_cfg.end())
|
||||
{
|
||||
new_cfg["pg_size"] = new_cfg["size"];
|
||||
new_cfg.erase("size");
|
||||
}
|
||||
|
||||
// --ec shortcut
|
||||
if (new_cfg.find("ec") != new_cfg.end())
|
||||
{
|
||||
if (new_cfg.find("scheme") != new_cfg.end() ||
|
||||
new_cfg.find("pg_size") != new_cfg.end() ||
|
||||
new_cfg.find("parity_chunks") != new_cfg.end())
|
||||
{
|
||||
return "--ec can't be used with --pg_size, --parity_chunks or --scheme";
|
||||
}
|
||||
// pg_size = N+K
|
||||
// parity_chunks = K
|
||||
uint64_t data_chunks = 0, parity_chunks = 0;
|
||||
char null_byte = 0;
|
||||
int ret = sscanf(new_cfg["ec"].string_value().c_str(), "%ju+%ju%c", &data_chunks, &parity_chunks, &null_byte);
|
||||
if (ret != 2 || !data_chunks || !parity_chunks)
|
||||
{
|
||||
return "--ec should be <N>+<K> format (<N>, <K> - numbers)";
|
||||
}
|
||||
new_cfg.erase("ec");
|
||||
new_cfg["scheme"] = "ec";
|
||||
new_cfg["pg_size"] = data_chunks+parity_chunks;
|
||||
new_cfg["parity_chunks"] = parity_chunks;
|
||||
}
|
||||
|
||||
if (old_cfg.is_null() && new_cfg["scheme"].string_value() == "")
|
||||
{
|
||||
// Default scheme
|
||||
new_cfg["scheme"] = "replicated";
|
||||
}
|
||||
if (new_cfg.find("pg_minsize") == new_cfg.end() && (old_cfg.is_null() || new_cfg.find("pg_size") != new_cfg.end()))
|
||||
{
|
||||
// Default pg_minsize
|
||||
if (new_cfg["scheme"] == "replicated")
|
||||
{
|
||||
// pg_minsize = (N+K > 2) ? 2 : 1
|
||||
new_cfg["pg_minsize"] = new_cfg["pg_size"].uint64_value() > 2 ? 2 : 1;
|
||||
}
|
||||
else // ec or xor
|
||||
{
|
||||
// pg_minsize = (K > 1) ? N + 1 : N
|
||||
new_cfg["pg_minsize"] = new_cfg["pg_size"].uint64_value() - new_cfg["parity_chunks"].uint64_value() +
|
||||
(new_cfg["parity_chunks"].uint64_value() > 1 ? 1 : 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Check integer values and unknown keys
|
||||
for (auto kv_it = new_cfg.begin(); kv_it != new_cfg.end(); )
|
||||
{
|
||||
auto & key = kv_it->first;
|
||||
auto & value = kv_it->second;
|
||||
if (key == "pg_size" || key == "parity_chunks" || key == "pg_minsize" ||
|
||||
key == "pg_count" || key == "max_osd_combinations" || key == "block_size" ||
|
||||
key == "bitmap_granularity" || key == "pg_stripe_size")
|
||||
{
|
||||
if (value.is_number() && value.uint64_value() != value.number_value() ||
|
||||
value.is_string() && !value.uint64_value() && value.string_value() != "0")
|
||||
{
|
||||
return key+" must be a non-negative integer";
|
||||
}
|
||||
value = value.uint64_value();
|
||||
}
|
||||
else if (key == "name" || key == "scheme" || key == "immediate_commit" ||
|
||||
key == "failure_domain" || key == "root_node" || key == "scrub_interval" || key == "used_for_fs")
|
||||
{
|
||||
// OK
|
||||
}
|
||||
else if (key == "osd_tags" || key == "primary_affinity_tags")
|
||||
{
|
||||
if (value.is_string())
|
||||
{
|
||||
value = explode(",", value.string_value(), true);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Unknown parameter
|
||||
new_cfg.erase(kv_it++);
|
||||
continue;
|
||||
}
|
||||
kv_it++;
|
||||
}
|
||||
|
||||
// Merge with the old config
|
||||
if (!old_cfg.is_null())
|
||||
{
|
||||
for (auto & kv: old_cfg.object_items())
|
||||
{
|
||||
if (new_cfg.find(kv.first) == new_cfg.end())
|
||||
{
|
||||
new_cfg[kv.first] = kv.second;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check after merging
|
||||
if (new_cfg["scheme"] != "ec")
|
||||
{
|
||||
new_cfg.erase("parity_chunks");
|
||||
}
|
||||
if (new_cfg.find("used_for_fs") != new_cfg.end() && new_cfg["used_for_fs"].string_value() == "")
|
||||
{
|
||||
new_cfg.erase("used_for_fs");
|
||||
}
|
||||
|
||||
// Prevent autovivification of object keys. Now we don't modify the config, we just check it
|
||||
json11::Json cfg = new_cfg;
|
||||
|
||||
// Validate changes
|
||||
if (!old_cfg.is_null() && !force)
|
||||
{
|
||||
if (old_cfg["scheme"] != cfg["scheme"])
|
||||
{
|
||||
return "Changing scheme for an existing pool will lead to data loss. Use --force to proceed";
|
||||
}
|
||||
if (etcd_state_client_t::parse_scheme(old_cfg["scheme"].string_value()) == POOL_SCHEME_EC)
|
||||
{
|
||||
uint64_t old_data_chunks = old_cfg["pg_size"].uint64_value() - old_cfg["parity_chunks"].uint64_value();
|
||||
uint64_t new_data_chunks = cfg["pg_size"].uint64_value() - cfg["parity_chunks"].uint64_value();
|
||||
if (old_data_chunks != new_data_chunks)
|
||||
{
|
||||
return "Changing EC data chunk count for an existing pool will lead to data loss. Use --force to proceed";
|
||||
}
|
||||
}
|
||||
if (old_cfg["block_size"] != cfg["block_size"] ||
|
||||
old_cfg["bitmap_granularity"] != cfg["bitmap_granularity"] ||
|
||||
old_cfg["immediate_commit"] != cfg["immediate_commit"])
|
||||
{
|
||||
return "Changing block_size, bitmap_granularity or immediate_commit"
|
||||
" for an existing pool will lead to incomplete PGs. Use --force to proceed";
|
||||
}
|
||||
if (old_cfg["pg_stripe_size"] != cfg["pg_stripe_size"])
|
||||
{
|
||||
return "Changing pg_stripe_size for an existing pool will lead to data loss. Use --force to proceed";
|
||||
}
|
||||
}
|
||||
|
||||
// Validate values
|
||||
if (cfg["name"].string_value() == "")
|
||||
{
|
||||
return "Non-empty pool name is required";
|
||||
}
|
||||
|
||||
// scheme
|
||||
auto scheme = etcd_state_client_t::parse_scheme(cfg["scheme"].string_value());
|
||||
if (!scheme)
|
||||
{
|
||||
return "Scheme must be one of \"replicated\", \"ec\" or \"xor\"";
|
||||
}
|
||||
|
||||
// pg_size
|
||||
auto pg_size = cfg["pg_size"].uint64_value();
|
||||
if (!pg_size)
|
||||
{
|
||||
return "Non-zero PG size is required";
|
||||
}
|
||||
if (scheme != POOL_SCHEME_REPLICATED && pg_size < 3)
|
||||
{
|
||||
return "PG size can't be smaller than 3 for EC/XOR pools";
|
||||
}
|
||||
if (pg_size > 256)
|
||||
{
|
||||
return "PG size can't be greater than 256";
|
||||
}
|
||||
|
||||
// parity_chunks
|
||||
uint64_t parity_chunks = 1;
|
||||
if (scheme == POOL_SCHEME_EC)
|
||||
{
|
||||
parity_chunks = cfg["parity_chunks"].uint64_value();
|
||||
if (!parity_chunks)
|
||||
{
|
||||
return "Non-zero parity_chunks is required";
|
||||
}
|
||||
if (parity_chunks > pg_size-2)
|
||||
{
|
||||
return "parity_chunks can't be greater than "+std::to_string(pg_size-2)+" (PG size - 2)";
|
||||
}
|
||||
}
|
||||
|
||||
// pg_minsize
|
||||
auto pg_minsize = cfg["pg_minsize"].uint64_value();
|
||||
if (!pg_minsize)
|
||||
{
|
||||
return "Non-zero pg_minsize is required";
|
||||
}
|
||||
else if (pg_minsize > pg_size)
|
||||
{
|
||||
return "pg_minsize can't be greater than "+std::to_string(pg_size)+" (PG size)";
|
||||
}
|
||||
else if (scheme != POOL_SCHEME_REPLICATED && pg_minsize < pg_size-parity_chunks)
|
||||
{
|
||||
return "pg_minsize can't be smaller than "+std::to_string(pg_size-parity_chunks)+
|
||||
" (pg_size - parity_chunks) for XOR/EC pool";
|
||||
}
|
||||
|
||||
// pg_count
|
||||
if (!cfg["pg_count"].uint64_value())
|
||||
{
|
||||
return "Non-zero pg_count is required";
|
||||
}
|
||||
|
||||
// max_osd_combinations
|
||||
if (!cfg["max_osd_combinations"].is_null() && cfg["max_osd_combinations"].uint64_value() < 100)
|
||||
{
|
||||
return "max_osd_combinations must be at least 100, but it is "+cfg["max_osd_combinations"].as_string();
|
||||
}
|
||||
|
||||
// block_size
|
||||
auto block_size = cfg["block_size"].uint64_value();
|
||||
if (!cfg["block_size"].is_null() && ((block_size & (block_size-1)) ||
|
||||
block_size < MIN_DATA_BLOCK_SIZE || block_size > MAX_DATA_BLOCK_SIZE))
|
||||
{
|
||||
return "block_size must be a power of two between "+std::to_string(MIN_DATA_BLOCK_SIZE)+
|
||||
" and "+std::to_string(MAX_DATA_BLOCK_SIZE)+", but it is "+std::to_string(block_size);
|
||||
}
|
||||
block_size = (block_size ? block_size : global_block_size);
|
||||
|
||||
// bitmap_granularity
|
||||
auto bitmap_granularity = cfg["bitmap_granularity"].uint64_value();
|
||||
if (!cfg["bitmap_granularity"].is_null() && (!bitmap_granularity || (bitmap_granularity % 512)))
|
||||
{
|
||||
return "bitmap_granularity must be a multiple of 512, but it is "+std::to_string(bitmap_granularity);
|
||||
}
|
||||
bitmap_granularity = (bitmap_granularity ? bitmap_granularity : global_bitmap_granularity);
|
||||
if (block_size % bitmap_granularity)
|
||||
{
|
||||
return "bitmap_granularity must divide data block size ("+std::to_string(block_size)+"), but it is "+std::to_string(bitmap_granularity);
|
||||
}
|
||||
|
||||
// immediate_commit
|
||||
if (!cfg["immediate_commit"].is_null() && !etcd_state_client_t::parse_immediate_commit(cfg["immediate_commit"].string_value()))
|
||||
{
|
||||
return "immediate_commit must be one of \"all\", \"small\", or \"none\", but it is "+cfg["immediate_commit"].as_string();
|
||||
}
|
||||
|
||||
// scrub_interval
|
||||
if (!cfg["scrub_interval"].is_null())
|
||||
{
|
||||
bool ok;
|
||||
parse_time(cfg["scrub_interval"].string_value(), &ok);
|
||||
if (!ok)
|
||||
{
|
||||
return "scrub_interval must be a time interval (number + unit s/m/h/d/M/y), but it is "+cfg["scrub_interval"].as_string();
|
||||
}
|
||||
}
|
||||
|
||||
return "";
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2024
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "json11/json11.hpp"
|
||||
#include <stdint.h>
|
||||
|
||||
std::string validate_pool_config(json11::Json::object & new_cfg, json11::Json old_cfg,
|
||||
uint64_t global_block_size, uint64_t global_bitmap_granularity, bool force);
|
|
@ -0,0 +1,622 @@
|
|||
// Copyright (c) MIND Software LLC, 2023 (info@mindsw.io)
|
||||
// I accept Vitastor CLA: see CLA-en.md for details
|
||||
// Copyright (c) Vitaliy Filippov, 2024
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <ctype.h>
|
||||
#include "cli.h"
|
||||
#include "cli_pool_cfg.h"
|
||||
#include "cluster_client.h"
|
||||
#include "epoll_manager.h"
|
||||
#include "pg_states.h"
|
||||
#include "str_util.h"
|
||||
|
||||
struct pool_creator_t
|
||||
{
|
||||
cli_tool_t *parent;
|
||||
json11::Json::object cfg;
|
||||
|
||||
bool force = false;
|
||||
bool wait = false;
|
||||
|
||||
int state = 0;
|
||||
cli_result_t result;
|
||||
|
||||
struct {
|
||||
uint32_t retries = 5;
|
||||
uint32_t interval = 0;
|
||||
bool passed = false;
|
||||
} create_check;
|
||||
|
||||
uint64_t new_id = 1;
|
||||
uint64_t new_pools_mod_rev;
|
||||
json11::Json state_node_tree;
|
||||
json11::Json new_pools;
|
||||
|
||||
bool is_done() { return state == 100; }
|
||||
|
||||
void loop()
|
||||
{
|
||||
if (state == 1)
|
||||
goto resume_1;
|
||||
else if (state == 2)
|
||||
goto resume_2;
|
||||
else if (state == 3)
|
||||
goto resume_3;
|
||||
else if (state == 4)
|
||||
goto resume_4;
|
||||
else if (state == 5)
|
||||
goto resume_5;
|
||||
else if (state == 6)
|
||||
goto resume_6;
|
||||
else if (state == 7)
|
||||
goto resume_7;
|
||||
else if (state == 8)
|
||||
goto resume_8;
|
||||
|
||||
// Validate pool parameters
|
||||
result.text = validate_pool_config(cfg, json11::Json(), parent->cli->st_cli.global_block_size,
|
||||
parent->cli->st_cli.global_bitmap_granularity, force);
|
||||
if (result.text != "")
|
||||
{
|
||||
result.err = EINVAL;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
state = 1;
|
||||
resume_1:
|
||||
// If not forced, check that we have enough osds for pg_size
|
||||
if (!force)
|
||||
{
|
||||
// Get node_placement configuration from etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/node_placement") },
|
||||
} }
|
||||
},
|
||||
} },
|
||||
});
|
||||
|
||||
state = 2;
|
||||
resume_2:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Get state_node_tree based on node_placement and osd peer states
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
|
||||
state_node_tree = get_state_node_tree(kv.value.object_items());
|
||||
}
|
||||
|
||||
// Skip tag checks, if pool has none
|
||||
if (cfg["osd_tags"].array_items().size())
|
||||
{
|
||||
// Get osd configs (for tags) of osds in state_node_tree
|
||||
{
|
||||
json11::Json::array osd_configs;
|
||||
for (auto osd_num: state_node_tree["osds"].array_items())
|
||||
{
|
||||
osd_configs.push_back(json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/osd/"+osd_num.as_string()) },
|
||||
} }
|
||||
});
|
||||
}
|
||||
parent->etcd_txn(json11::Json::object { { "success", osd_configs, }, });
|
||||
}
|
||||
|
||||
state = 3;
|
||||
resume_3:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter out osds from state_node_tree based on pool/osd tags
|
||||
{
|
||||
std::vector<json11::Json> osd_configs;
|
||||
for (auto & ocr: parent->etcd_result["responses"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(ocr["response_range"]["kvs"][0]);
|
||||
osd_configs.push_back(kv.value);
|
||||
}
|
||||
state_node_tree = filter_state_node_tree_by_tags(state_node_tree, osd_configs);
|
||||
}
|
||||
}
|
||||
|
||||
// Get stats (for block_size, bitmap_granularity, ...) of osds in state_node_tree
|
||||
{
|
||||
json11::Json::array osd_stats;
|
||||
|
||||
for (auto osd_num: state_node_tree["osds"].array_items())
|
||||
{
|
||||
osd_stats.push_back(json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/osd/stats/"+osd_num.as_string()) },
|
||||
} }
|
||||
});
|
||||
}
|
||||
|
||||
parent->etcd_txn(json11::Json::object { { "success", osd_stats, }, });
|
||||
}
|
||||
|
||||
state = 4;
|
||||
resume_4:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Filter osds from state_node_tree based on pool parameters and osd stats
|
||||
{
|
||||
std::vector<json11::Json> osd_stats;
|
||||
for (auto & ocr: parent->etcd_result["responses"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(ocr["response_range"]["kvs"][0]);
|
||||
osd_stats.push_back(kv.value);
|
||||
}
|
||||
state_node_tree = filter_state_node_tree_by_stats(state_node_tree, osd_stats);
|
||||
}
|
||||
|
||||
// Check that pg_size <= max_pg_size
|
||||
{
|
||||
auto failure_domain = cfg["failure_domain"].string_value() == ""
|
||||
? "host" : cfg["failure_domain"].string_value();
|
||||
uint64_t max_pg_size = get_max_pg_size(state_node_tree["nodes"].object_items(),
|
||||
failure_domain, cfg["root_node"].string_value());
|
||||
|
||||
if (cfg["pg_size"].uint64_value() > max_pg_size)
|
||||
{
|
||||
result = (cli_result_t){
|
||||
.err = EINVAL,
|
||||
.text =
|
||||
"There are "+std::to_string(max_pg_size)+" \""+failure_domain+"\" failure domains with OSDs matching tags and"
|
||||
" block_size/bitmap_granularity/immediate_commit parameters, but you want to create a"
|
||||
" pool with "+cfg["pg_size"].as_string()+" OSDs from different failure domains in a PG."
|
||||
" Change parameters or add --force if you want to create a degraded pool and add OSDs later."
|
||||
};
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Create pool
|
||||
state = 5;
|
||||
resume_5:
|
||||
// Get pools from etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
} }
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 6;
|
||||
resume_6:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
{
|
||||
// Add new pool
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
|
||||
new_pools = create_pool(kv);
|
||||
if (new_pools.is_string())
|
||||
{
|
||||
result = (cli_result_t){ .err = EEXIST, .text = new_pools.string_value() };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
new_pools_mod_rev = kv.mod_revision;
|
||||
}
|
||||
// Update pools in etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "compare", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "target", "MOD" },
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "result", "LESS" },
|
||||
{ "mod_revision", new_pools_mod_rev+1 },
|
||||
}
|
||||
} },
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "value", base64_encode(new_pools.dump()) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 7;
|
||||
resume_7:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Perform final create-check
|
||||
create_check.interval = parent->cli->config["mon_change_timeout"].uint64_value();
|
||||
if (!create_check.interval)
|
||||
create_check.interval = 1000;
|
||||
|
||||
state = 8;
|
||||
resume_8:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
|
||||
// Unless forced, check that pool was created and is active
|
||||
if (!wait)
|
||||
{
|
||||
create_check.passed = true;
|
||||
}
|
||||
else if (create_check.retries)
|
||||
{
|
||||
create_check.retries--;
|
||||
parent->waiting++;
|
||||
parent->epmgr->tfd->set_timer(create_check.interval, false, [this](int timer_id)
|
||||
{
|
||||
if (parent->cli->st_cli.pool_config.find(new_id) != parent->cli->st_cli.pool_config.end())
|
||||
{
|
||||
auto & pool_cfg = parent->cli->st_cli.pool_config[new_id];
|
||||
create_check.passed = pool_cfg.real_pg_count > 0;
|
||||
for (auto pg_it = pool_cfg.pg_config.begin(); pg_it != pool_cfg.pg_config.end(); pg_it++)
|
||||
{
|
||||
if (!(pg_it->second.cur_state & PG_ACTIVE))
|
||||
{
|
||||
create_check.passed = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (create_check.passed)
|
||||
create_check.retries = 0;
|
||||
}
|
||||
parent->waiting--;
|
||||
parent->ringloop->wakeup();
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!create_check.passed)
|
||||
{
|
||||
result = (cli_result_t) {
|
||||
.err = EAGAIN,
|
||||
.text = "Pool "+cfg["name"].string_value()+" was created, but failed to become active."
|
||||
" This may indicate that cluster state has changed while the pool was being created."
|
||||
" Please check the current state and adjust the pool configuration if necessary.",
|
||||
};
|
||||
}
|
||||
else
|
||||
{
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Pool "+cfg["name"].string_value()+" created",
|
||||
.data = new_pools[std::to_string(new_id)],
|
||||
};
|
||||
}
|
||||
state = 100;
|
||||
}
|
||||
|
||||
// Returns a JSON object of form {"nodes": {...}, "osds": [...]} that
|
||||
// contains: all nodes (osds, hosts, ...) based on node_placement config
|
||||
// and current peer state, and a list of active peer osds.
|
||||
json11::Json get_state_node_tree(json11::Json::object node_placement)
|
||||
{
|
||||
// Erase non-peer osd nodes from node_placement
|
||||
for (auto np_it = node_placement.begin(); np_it != node_placement.end();)
|
||||
{
|
||||
// Numeric nodes are osds
|
||||
osd_num_t osd_num = stoull_full(np_it->first);
|
||||
|
||||
// If node is osd and it is not in peer states, erase it
|
||||
if (osd_num > 0 &&
|
||||
parent->cli->st_cli.peer_states.find(osd_num) == parent->cli->st_cli.peer_states.end())
|
||||
{
|
||||
node_placement.erase(np_it++);
|
||||
}
|
||||
else
|
||||
np_it++;
|
||||
}
|
||||
|
||||
// List of peer osds
|
||||
std::vector<std::string> peer_osds;
|
||||
|
||||
// Record peer osds and add missing osds/hosts to np
|
||||
for (auto & ps: parent->cli->st_cli.peer_states)
|
||||
{
|
||||
std::string osd_num = std::to_string(ps.first);
|
||||
|
||||
// Record peer osd
|
||||
peer_osds.push_back(osd_num);
|
||||
|
||||
// Add osd, if necessary
|
||||
if (node_placement.find(osd_num) == node_placement.end())
|
||||
{
|
||||
std::string osd_host = ps.second["host"].as_string();
|
||||
|
||||
// Add host, if necessary
|
||||
if (node_placement.find(osd_host) == node_placement.end())
|
||||
{
|
||||
node_placement[osd_host] = json11::Json::object {
|
||||
{ "level", "host" }
|
||||
};
|
||||
}
|
||||
|
||||
node_placement[osd_num] = json11::Json::object {
|
||||
{ "parent", osd_host }
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return json11::Json::object { { "osds", peer_osds }, { "nodes", node_placement } };
|
||||
}
|
||||
|
||||
// Returns new state_node_tree based on given state_node_tree with osds
|
||||
// filtered out by tags in given osd_configs and current pool config.
|
||||
// Requires: state_node_tree["osds"] must match osd_configs 1-1
|
||||
json11::Json filter_state_node_tree_by_tags(const json11::Json & state_node_tree, std::vector<json11::Json> & osd_configs)
|
||||
{
|
||||
auto & osds = state_node_tree["osds"].array_items();
|
||||
|
||||
// Accepted state_node_tree nodes
|
||||
auto accepted_nodes = state_node_tree["nodes"].object_items();
|
||||
|
||||
// List of accepted osds
|
||||
std::vector<std::string> accepted_osds;
|
||||
|
||||
for (size_t i = 0; i < osd_configs.size(); i++)
|
||||
{
|
||||
auto & oc = osd_configs[i].object_items();
|
||||
|
||||
// Get osd number
|
||||
auto osd_num = osds[i].as_string();
|
||||
|
||||
// We need tags in config to check against pool tags
|
||||
if (oc.find("tags") == oc.end())
|
||||
{
|
||||
// Exclude osd from state_node_tree nodes
|
||||
accepted_nodes.erase(osd_num);
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
// If all pool tags are in osd tags, accept osd
|
||||
if (all_in_tags(osd_configs[i]["tags"], cfg["osd_tags"]))
|
||||
{
|
||||
accepted_osds.push_back(osd_num);
|
||||
}
|
||||
// Otherwise, exclude osd
|
||||
else
|
||||
{
|
||||
// Exclude osd from state_node_tree nodes
|
||||
accepted_nodes.erase(osd_num);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return json11::Json::object { { "osds", accepted_osds }, { "nodes", accepted_nodes } };
|
||||
}
|
||||
|
||||
// Returns new state_node_tree based on given state_node_tree with osds
|
||||
// filtered out by stats parameters (block_size, bitmap_granularity) in
|
||||
// given osd_stats and current pool config.
|
||||
// Requires: state_node_tree["osds"] must match osd_stats 1-1
|
||||
json11::Json filter_state_node_tree_by_stats(const json11::Json & state_node_tree, std::vector<json11::Json> & osd_stats)
|
||||
{
|
||||
auto & osds = state_node_tree["osds"].array_items();
|
||||
|
||||
// Accepted state_node_tree nodes
|
||||
auto accepted_nodes = state_node_tree["nodes"].object_items();
|
||||
|
||||
// List of accepted osds
|
||||
std::vector<std::string> accepted_osds;
|
||||
|
||||
uint64_t p_block_size = cfg["block_size"].uint64_value()
|
||||
? cfg["block_size"].uint64_value()
|
||||
: parent->cli->st_cli.global_block_size;
|
||||
uint64_t p_bitmap_granularity = cfg["bitmap_granularity"].uint64_value()
|
||||
? cfg["bitmap_granularity"].uint64_value()
|
||||
: parent->cli->st_cli.global_bitmap_granularity;
|
||||
uint32_t p_immediate_commit = cfg["immediate_commit"].is_string()
|
||||
? etcd_state_client_t::parse_immediate_commit(cfg["immediate_commit"].string_value())
|
||||
: parent->cli->st_cli.global_immediate_commit;
|
||||
|
||||
for (size_t i = 0; i < osd_stats.size(); i++)
|
||||
{
|
||||
auto & os = osd_stats[i];
|
||||
// Get osd number
|
||||
auto osd_num = osds[i].as_string();
|
||||
if (!os["data_block_size"].is_null() && os["data_block_size"] != p_block_size ||
|
||||
!os["bitmap_granularity"].is_null() && os["bitmap_granularity"] != p_bitmap_granularity ||
|
||||
!os["immediate_commit"].is_null() &&
|
||||
etcd_state_client_t::parse_immediate_commit(os["immediate_commit"].string_value()) < p_immediate_commit)
|
||||
{
|
||||
accepted_nodes.erase(osd_num);
|
||||
}
|
||||
else
|
||||
{
|
||||
accepted_osds.push_back(osd_num);
|
||||
}
|
||||
}
|
||||
|
||||
return json11::Json::object { { "osds", accepted_osds }, { "nodes", accepted_nodes } };
|
||||
}
|
||||
|
||||
// Returns maximum pg_size possible for given node_tree and failure_domain, starting at parent_node
|
||||
uint64_t get_max_pg_size(json11::Json::object node_tree, const std::string & level, const std::string & parent_node)
|
||||
{
|
||||
uint64_t max_pg_sz = 0;
|
||||
|
||||
std::vector<std::string> nodes;
|
||||
|
||||
// Check if parent node is an osd (numeric)
|
||||
if (parent_node != "" && stoull_full(parent_node))
|
||||
{
|
||||
// Add it to node list if osd is in node tree
|
||||
if (node_tree.find(parent_node) != node_tree.end())
|
||||
nodes.push_back(parent_node);
|
||||
}
|
||||
// If parent node given, ...
|
||||
else if (parent_node != "")
|
||||
{
|
||||
// ... look for children nodes of this parent
|
||||
for (auto & sn: node_tree)
|
||||
{
|
||||
auto & props = sn.second.object_items();
|
||||
|
||||
auto parent_prop = props.find("parent");
|
||||
if (parent_prop != props.end() && (parent_prop->second.as_string() == parent_node))
|
||||
{
|
||||
nodes.push_back(sn.first);
|
||||
|
||||
// If we're not looking for all osds, we only need a single
|
||||
// child osd node
|
||||
if (level != "osd" && stoull_full(sn.first))
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// No parent node given, and we're not looking for all osds
|
||||
else if (level != "osd")
|
||||
{
|
||||
// ... look for all level nodes
|
||||
for (auto & sn: node_tree)
|
||||
{
|
||||
auto & props = sn.second.object_items();
|
||||
|
||||
auto level_prop = props.find("level");
|
||||
if (level_prop != props.end() && (level_prop->second.as_string() == level))
|
||||
{
|
||||
nodes.push_back(sn.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Otherwise, ...
|
||||
else
|
||||
{
|
||||
// ... we're looking for osd nodes only
|
||||
for (auto & sn: node_tree)
|
||||
{
|
||||
if (stoull_full(sn.first))
|
||||
{
|
||||
nodes.push_back(sn.first);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process gathered nodes
|
||||
for (auto & node: nodes)
|
||||
{
|
||||
// Check for osd node, return constant max size
|
||||
if (stoull_full(node))
|
||||
{
|
||||
max_pg_sz += 1;
|
||||
}
|
||||
// Otherwise, ...
|
||||
else
|
||||
{
|
||||
// ... exclude parent node from tree, and ...
|
||||
node_tree.erase(parent_node);
|
||||
|
||||
// ... descend onto the resulting tree
|
||||
max_pg_sz += get_max_pg_size(node_tree, level, node);
|
||||
}
|
||||
}
|
||||
|
||||
return max_pg_sz;
|
||||
}
|
||||
|
||||
json11::Json create_pool(const etcd_kv_t & kv)
|
||||
{
|
||||
for (auto & p: kv.value.object_items())
|
||||
{
|
||||
// ID
|
||||
uint64_t pool_id = stoull_full(p.first);
|
||||
new_id = std::max(pool_id+1, new_id);
|
||||
// Name
|
||||
if (p.second["name"].string_value() == cfg["name"].string_value())
|
||||
{
|
||||
return "Pool with name \""+cfg["name"].string_value()+"\" already exists (ID "+std::to_string(pool_id)+")";
|
||||
}
|
||||
}
|
||||
auto res = kv.value.object_items();
|
||||
res[std::to_string(new_id)] = cfg;
|
||||
return res;
|
||||
}
|
||||
|
||||
// Checks whether tags2 tags are all in tags1 tags
|
||||
bool all_in_tags(json11::Json tags1, json11::Json tags2)
|
||||
{
|
||||
if (!tags2.is_array())
|
||||
{
|
||||
tags2 = json11::Json::array{ tags2.string_value() };
|
||||
}
|
||||
if (!tags1.is_array())
|
||||
{
|
||||
tags1 = json11::Json::array{ tags1.string_value() };
|
||||
}
|
||||
for (auto & tag2: tags2.array_items())
|
||||
{
|
||||
bool found = false;
|
||||
for (auto & tag1: tags1.array_items())
|
||||
{
|
||||
if (tag1 == tag2)
|
||||
{
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
};
|
||||
|
||||
std::function<bool(cli_result_t &)> cli_tool_t::start_pool_create(json11::Json cfg)
|
||||
{
|
||||
auto pool_creator = new pool_creator_t();
|
||||
pool_creator->parent = this;
|
||||
pool_creator->cfg = cfg.object_items();
|
||||
pool_creator->force = cfg["force"].bool_value();
|
||||
pool_creator->wait = cfg["wait"].bool_value();
|
||||
return [pool_creator](cli_result_t & result)
|
||||
{
|
||||
pool_creator->loop();
|
||||
if (pool_creator->is_done())
|
||||
{
|
||||
result = pool_creator->result;
|
||||
delete pool_creator;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
|
@ -0,0 +1,723 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <algorithm>
|
||||
#include "cli.h"
|
||||
#include "cluster_client.h"
|
||||
#include "str_util.h"
|
||||
#include "pg_states.h"
|
||||
|
||||
// List pools with space statistics
|
||||
// - df - minimal list with % used space
|
||||
// - pool-ls - same but with PG state and recovery %
|
||||
// - pool-ls -l - same but also include I/O statistics
|
||||
// - pool-ls --detail - use list format, include PG states, I/O stats and all pool parameters
|
||||
struct pool_lister_t
|
||||
{
|
||||
cli_tool_t *parent;
|
||||
std::string sort_field;
|
||||
std::set<std::string> only_names;
|
||||
bool reverse = false;
|
||||
int max_count = 0;
|
||||
bool show_recovery = false;
|
||||
bool show_stats = false;
|
||||
bool detailed = false;
|
||||
|
||||
int state = 0;
|
||||
cli_result_t result;
|
||||
std::map<pool_id_t, json11::Json::object> pool_stats;
|
||||
struct io_stats_t
|
||||
{
|
||||
uint64_t count = 0;
|
||||
uint64_t read_iops = 0;
|
||||
uint64_t read_bps = 0;
|
||||
uint64_t read_lat = 0;
|
||||
uint64_t write_iops = 0;
|
||||
uint64_t write_bps = 0;
|
||||
uint64_t write_lat = 0;
|
||||
uint64_t delete_iops = 0;
|
||||
uint64_t delete_bps = 0;
|
||||
uint64_t delete_lat = 0;
|
||||
};
|
||||
struct object_counts_t
|
||||
{
|
||||
uint64_t object_count = 0;
|
||||
uint64_t misplaced_count = 0;
|
||||
uint64_t degraded_count = 0;
|
||||
uint64_t incomplete_count = 0;
|
||||
};
|
||||
|
||||
bool is_done()
|
||||
{
|
||||
return state == 100;
|
||||
}
|
||||
|
||||
void get_pool_stats(int base_state)
|
||||
{
|
||||
if (state == base_state+1)
|
||||
goto resume_1;
|
||||
// Space statistics - pool/stats/<pool>
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pool/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pool/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/osd/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/osd/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/config/pools"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = base_state+1;
|
||||
resume_1:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
auto space_info = parent->etcd_result;
|
||||
auto config_pools = space_info["responses"][2]["response_range"]["kvs"][0];
|
||||
if (!config_pools.is_null())
|
||||
{
|
||||
config_pools = parent->cli->st_cli.parse_etcd_kv(config_pools).value;
|
||||
}
|
||||
for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// pool ID
|
||||
pool_id_t pool_id;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
|
||||
if (scanned != 1 || !pool_id || pool_id >= POOL_ID_MAX)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
// pool/stats/<N>
|
||||
pool_stats[pool_id] = kv.value.object_items();
|
||||
}
|
||||
std::map<pool_id_t, uint64_t> osd_free;
|
||||
for (auto & kv_item: space_info["responses"][1]["response_range"]["kvs"].array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// osd ID
|
||||
osd_num_t osd_num;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/osd/stats/%ju%c", &osd_num, &null_byte);
|
||||
if (scanned != 1 || !osd_num || osd_num >= POOL_ID_MAX)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
// osd/stats/<N>::free
|
||||
osd_free[osd_num] = kv.value["free"].uint64_value();
|
||||
}
|
||||
// Calculate max_avail for each pool
|
||||
for (auto & pp: parent->cli->st_cli.pool_config)
|
||||
{
|
||||
auto & pool_cfg = pp.second;
|
||||
uint64_t pool_avail = UINT64_MAX;
|
||||
std::map<osd_num_t, uint64_t> pg_per_osd;
|
||||
bool active = pool_cfg.real_pg_count > 0;
|
||||
uint64_t pg_states = 0;
|
||||
for (auto & pgp: pool_cfg.pg_config)
|
||||
{
|
||||
if (!(pgp.second.cur_state & PG_ACTIVE))
|
||||
{
|
||||
active = false;
|
||||
}
|
||||
pg_states |= pgp.second.cur_state;
|
||||
for (auto pg_osd: pgp.second.target_set)
|
||||
{
|
||||
if (pg_osd != 0)
|
||||
{
|
||||
pg_per_osd[pg_osd]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (auto pg_per_pair: pg_per_osd)
|
||||
{
|
||||
uint64_t pg_free = osd_free[pg_per_pair.first] * pool_cfg.real_pg_count / pg_per_pair.second;
|
||||
if (pool_avail > pg_free)
|
||||
{
|
||||
pool_avail = pg_free;
|
||||
}
|
||||
}
|
||||
if (pool_avail == UINT64_MAX)
|
||||
{
|
||||
pool_avail = 0;
|
||||
}
|
||||
if (pool_cfg.scheme != POOL_SCHEME_REPLICATED)
|
||||
{
|
||||
pool_avail *= (pool_cfg.pg_size - pool_cfg.parity_chunks);
|
||||
}
|
||||
// incomplete > has_incomplete > degraded > has_degraded > has_misplaced
|
||||
std::string status;
|
||||
if (!active)
|
||||
status = "inactive";
|
||||
else if (pg_states & PG_INCOMPLETE)
|
||||
status = "incomplete";
|
||||
else if (pg_states & PG_HAS_INCOMPLETE)
|
||||
status = "has_incomplete";
|
||||
else if (pg_states & PG_DEGRADED)
|
||||
status = "degraded";
|
||||
else if (pg_states & PG_HAS_DEGRADED)
|
||||
status = "has_degraded";
|
||||
else if (pg_states & PG_HAS_MISPLACED)
|
||||
status = "has_misplaced";
|
||||
else
|
||||
status = "active";
|
||||
pool_stats[pool_cfg.id] = json11::Json::object {
|
||||
{ "id", (uint64_t)pool_cfg.id },
|
||||
{ "name", pool_cfg.name },
|
||||
{ "status", status },
|
||||
{ "pg_count", pool_cfg.pg_count },
|
||||
{ "real_pg_count", pool_cfg.real_pg_count },
|
||||
{ "scheme_name", pool_cfg.scheme == POOL_SCHEME_REPLICATED
|
||||
? std::to_string(pool_cfg.pg_size)+"/"+std::to_string(pool_cfg.pg_minsize)
|
||||
: "EC "+std::to_string(pool_cfg.pg_size-pool_cfg.parity_chunks)+"+"+std::to_string(pool_cfg.parity_chunks) },
|
||||
{ "used_raw", (uint64_t)(pool_stats[pool_cfg.id]["used_raw_tb"].number_value() * ((uint64_t)1<<40)) },
|
||||
{ "total_raw", (uint64_t)(pool_stats[pool_cfg.id]["total_raw_tb"].number_value() * ((uint64_t)1<<40)) },
|
||||
{ "max_available", pool_avail },
|
||||
{ "raw_to_usable", pool_stats[pool_cfg.id]["raw_to_usable"].number_value() },
|
||||
{ "space_efficiency", pool_stats[pool_cfg.id]["space_efficiency"].number_value() },
|
||||
{ "pg_real_size", pool_stats[pool_cfg.id]["pg_real_size"].uint64_value() },
|
||||
{ "osd_count", pg_per_osd.size() },
|
||||
};
|
||||
}
|
||||
// Include full pool config
|
||||
for (auto & pp: config_pools.object_items())
|
||||
{
|
||||
if (!pp.second.is_object())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
auto pool_id = stoull_full(pp.first);
|
||||
auto & st = pool_stats[pool_id];
|
||||
for (auto & kv: pp.second.object_items())
|
||||
{
|
||||
if (st.find(kv.first) == st.end())
|
||||
st[kv.first] = kv.second;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void get_pg_stats(int base_state)
|
||||
{
|
||||
if (state == base_state+1)
|
||||
goto resume_1;
|
||||
// Space statistics - pool/stats/<pool>
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pg/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/pg/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = base_state+1;
|
||||
resume_1:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
auto pg_stats = parent->etcd_result["responses"][0]["response_range"]["kvs"];
|
||||
// Calculate recovery percent
|
||||
std::map<pool_id_t, object_counts_t> counts;
|
||||
for (auto & kv_item: pg_stats.array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// pool ID & pg number
|
||||
pool_id_t pool_id;
|
||||
pg_num_t pg_num = 0;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(),
|
||||
"/pg/stats/%u/%u%c", &pool_id, &pg_num, &null_byte);
|
||||
if (scanned != 2 || !pool_id || pool_id >= POOL_ID_MAX)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & cnt = counts[pool_id];
|
||||
cnt.object_count += kv.value["object_count"].uint64_value();
|
||||
cnt.misplaced_count += kv.value["misplaced_count"].uint64_value();
|
||||
cnt.degraded_count += kv.value["degraded_count"].uint64_value();
|
||||
cnt.incomplete_count += kv.value["incomplete_count"].uint64_value();
|
||||
}
|
||||
for (auto & pp: pool_stats)
|
||||
{
|
||||
auto & cnt = counts[pp.first];
|
||||
auto & st = pp.second;
|
||||
st["object_count"] = cnt.object_count;
|
||||
st["misplaced_count"] = cnt.misplaced_count;
|
||||
st["degraded_count"] = cnt.degraded_count;
|
||||
st["incomplete_count"] = cnt.incomplete_count;
|
||||
}
|
||||
}
|
||||
|
||||
void get_inode_stats(int base_state)
|
||||
{
|
||||
if (state == base_state+1)
|
||||
goto resume_1;
|
||||
// Space statistics - pool/stats/<pool>
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/inode/stats/"
|
||||
) },
|
||||
{ "range_end", base64_encode(
|
||||
parent->cli->st_cli.etcd_prefix+"/inode/stats0"
|
||||
) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = base_state+1;
|
||||
resume_1:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
auto inode_stats = parent->etcd_result["responses"][0]["response_range"]["kvs"];
|
||||
// Performance statistics
|
||||
std::map<pool_id_t, io_stats_t> pool_io;
|
||||
for (auto & kv_item: inode_stats.array_items())
|
||||
{
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
|
||||
// pool ID & inode number
|
||||
pool_id_t pool_id;
|
||||
inode_t only_inode_num;
|
||||
char null_byte = 0;
|
||||
int scanned = sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(),
|
||||
"/inode/stats/%u/%ju%c", &pool_id, &only_inode_num, &null_byte);
|
||||
if (scanned != 2 || !pool_id || pool_id >= POOL_ID_MAX || INODE_POOL(only_inode_num) != 0)
|
||||
{
|
||||
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
|
||||
continue;
|
||||
}
|
||||
auto & io = pool_io[pool_id];
|
||||
io.read_iops += kv.value["read"]["iops"].uint64_value();
|
||||
io.read_bps += kv.value["read"]["bps"].uint64_value();
|
||||
io.read_lat += kv.value["read"]["lat"].uint64_value();
|
||||
io.write_iops += kv.value["write"]["iops"].uint64_value();
|
||||
io.write_bps += kv.value["write"]["bps"].uint64_value();
|
||||
io.write_lat += kv.value["write"]["lat"].uint64_value();
|
||||
io.delete_iops += kv.value["delete"]["iops"].uint64_value();
|
||||
io.delete_bps += kv.value["delete"]["bps"].uint64_value();
|
||||
io.delete_lat += kv.value["delete"]["lat"].uint64_value();
|
||||
io.count++;
|
||||
}
|
||||
for (auto & pp: pool_stats)
|
||||
{
|
||||
auto & io = pool_io[pp.first];
|
||||
if (io.count > 0)
|
||||
{
|
||||
io.read_lat /= io.count;
|
||||
io.write_lat /= io.count;
|
||||
io.delete_lat /= io.count;
|
||||
}
|
||||
auto & st = pp.second;
|
||||
st["read_iops"] = io.read_iops;
|
||||
st["read_bps"] = io.read_bps;
|
||||
st["read_lat"] = io.read_lat;
|
||||
st["write_iops"] = io.write_iops;
|
||||
st["write_bps"] = io.write_bps;
|
||||
st["write_lat"] = io.write_lat;
|
||||
st["delete_iops"] = io.delete_iops;
|
||||
st["delete_bps"] = io.delete_bps;
|
||||
st["delete_lat"] = io.delete_lat;
|
||||
}
|
||||
}
|
||||
|
||||
json11::Json::array to_list()
|
||||
{
|
||||
json11::Json::array list;
|
||||
for (auto & kv: pool_stats)
|
||||
{
|
||||
if (!only_names.size())
|
||||
{
|
||||
list.push_back(kv.second);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (auto glob: only_names)
|
||||
{
|
||||
if (stupid_glob(kv.second["name"].string_value(), glob))
|
||||
{
|
||||
list.push_back(kv.second);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sort_field == "name" || sort_field == "scheme" ||
|
||||
sort_field == "scheme_name" || sort_field == "status")
|
||||
{
|
||||
std::sort(list.begin(), list.end(), [this](json11::Json a, json11::Json b)
|
||||
{
|
||||
auto av = a[sort_field].as_string();
|
||||
auto bv = b[sort_field].as_string();
|
||||
return reverse ? av > bv : av < bv;
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
std::sort(list.begin(), list.end(), [this](json11::Json a, json11::Json b)
|
||||
{
|
||||
auto av = a[sort_field].number_value();
|
||||
auto bv = b[sort_field].number_value();
|
||||
return reverse ? av > bv : av < bv;
|
||||
});
|
||||
}
|
||||
if (max_count > 0 && list.size() > max_count)
|
||||
{
|
||||
list.resize(max_count);
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
void loop()
|
||||
{
|
||||
if (state == 1)
|
||||
goto resume_1;
|
||||
if (state == 2)
|
||||
goto resume_2;
|
||||
if (state == 3)
|
||||
goto resume_3;
|
||||
if (state == 100)
|
||||
return;
|
||||
show_stats = show_stats || detailed;
|
||||
show_recovery = show_recovery || detailed;
|
||||
resume_1:
|
||||
get_pool_stats(0);
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (show_stats)
|
||||
{
|
||||
resume_2:
|
||||
get_inode_stats(1);
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
}
|
||||
if (show_recovery)
|
||||
{
|
||||
resume_3:
|
||||
get_pg_stats(2);
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
}
|
||||
if (parent->json_output)
|
||||
{
|
||||
// JSON output
|
||||
result.data = to_list();
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
json11::Json::array list;
|
||||
for (auto & kv: pool_stats)
|
||||
{
|
||||
auto & st = kv.second;
|
||||
double raw_to = st["raw_to_usable"].number_value();
|
||||
if (raw_to < 0.000001 && raw_to > -0.000001)
|
||||
raw_to = 1;
|
||||
st["pg_count_fmt"] = st["real_pg_count"] == st["pg_count"]
|
||||
? st["real_pg_count"].as_string()
|
||||
: st["real_pg_count"].as_string()+"->"+st["pg_count"].as_string();
|
||||
st["total_fmt"] = format_size(st["total_raw"].uint64_value() / raw_to);
|
||||
st["used_fmt"] = format_size(st["used_raw"].uint64_value() / raw_to);
|
||||
st["max_avail_fmt"] = format_size(st["max_available"].uint64_value());
|
||||
st["used_pct"] = format_q(st["total_raw"].uint64_value()
|
||||
? (100 - 100*st["max_available"].uint64_value() *
|
||||
st["raw_to_usable"].number_value() / st["total_raw"].uint64_value())
|
||||
: 100)+"%";
|
||||
st["eff_fmt"] = format_q(st["space_efficiency"].number_value()*100)+"%";
|
||||
if (show_stats)
|
||||
{
|
||||
st["read_bw"] = format_size(st["read_bps"].uint64_value())+"/s";
|
||||
st["write_bw"] = format_size(st["write_bps"].uint64_value())+"/s";
|
||||
st["delete_bw"] = format_size(st["delete_bps"].uint64_value())+"/s";
|
||||
st["read_iops"] = format_q(st["read_iops"].number_value());
|
||||
st["write_iops"] = format_q(st["write_iops"].number_value());
|
||||
st["delete_iops"] = format_q(st["delete_iops"].number_value());
|
||||
st["read_lat_f"] = format_lat(st["read_lat"].uint64_value());
|
||||
st["write_lat_f"] = format_lat(st["write_lat"].uint64_value());
|
||||
st["delete_lat_f"] = format_lat(st["delete_lat"].uint64_value());
|
||||
}
|
||||
if (show_recovery)
|
||||
{
|
||||
auto object_count = st["object_count"].uint64_value();
|
||||
auto recovery_pct = 100.0 * (object_count - (st["misplaced_count"].uint64_value() +
|
||||
st["degraded_count"].uint64_value() + st["incomplete_count"].uint64_value())) /
|
||||
(object_count ? object_count : 1);
|
||||
st["recovery_fmt"] = format_q(recovery_pct)+"%";
|
||||
}
|
||||
}
|
||||
if (detailed)
|
||||
{
|
||||
for (auto & kv: pool_stats)
|
||||
{
|
||||
auto & st = kv.second;
|
||||
auto total = st["object_count"].uint64_value();
|
||||
auto obj_size = st["block_size"].uint64_value();
|
||||
if (!obj_size)
|
||||
obj_size = parent->cli->st_cli.global_block_size;
|
||||
if (st["scheme"] == "ec")
|
||||
obj_size *= st["pg_size"].uint64_value() - st["parity_chunks"].uint64_value();
|
||||
else if (st["scheme"] == "xor")
|
||||
obj_size *= st["pg_size"].uint64_value() - 1;
|
||||
auto n = st["misplaced_count"].uint64_value();
|
||||
if (n > 0)
|
||||
st["misplaced_fmt"] = format_size(n * obj_size) + " / " + format_q(100.0 * n / total);
|
||||
n = st["degraded_count"].uint64_value();
|
||||
if (n > 0)
|
||||
st["degraded_fmt"] = format_size(n * obj_size) + " / " + format_q(100.0 * n / total);
|
||||
n = st["incomplete_count"].uint64_value();
|
||||
if (n > 0)
|
||||
st["incomplete_fmt"] = format_size(n * obj_size) + " / " + format_q(100.0 * n / total);
|
||||
st["read_fmt"] = st["read_bw"].string_value()+", "+st["read_iops"].string_value()+" op/s, "+
|
||||
st["read_lat_f"].string_value()+" lat";
|
||||
st["write_fmt"] = st["write_bw"].string_value()+", "+st["write_iops"].string_value()+" op/s, "+
|
||||
st["write_lat_f"].string_value()+" lat";
|
||||
st["delete_fmt"] = st["delete_bw"].string_value()+", "+st["delete_iops"].string_value()+" op/s, "+
|
||||
st["delete_lat_f"].string_value()+" lat";
|
||||
if (st["scheme"] == "replicated")
|
||||
st["scheme_name"] = "x"+st["pg_size"].as_string();
|
||||
if (st["failure_domain"].string_value() == "")
|
||||
st["failure_domain"] = "host";
|
||||
st["osd_tags_fmt"] = implode(", ", st["osd_tags"]);
|
||||
st["primary_affinity_tags_fmt"] = implode(", ", st["primary_affinity_tags"]);
|
||||
if (st["block_size"].uint64_value())
|
||||
st["block_size_fmt"] = format_size(st["block_size"].uint64_value());
|
||||
if (st["bitmap_granularity"].uint64_value())
|
||||
st["bitmap_granularity_fmt"] = format_size(st["bitmap_granularity"].uint64_value());
|
||||
}
|
||||
// All pool parameters are only displayed in the "detailed" mode
|
||||
// because there's too many of them to show them in table
|
||||
auto cols = std::vector<std::pair<std::string, std::string>>{
|
||||
{ "name", "Name" },
|
||||
{ "id", "ID" },
|
||||
{ "scheme_name", "Scheme" },
|
||||
{ "used_for_fs", "Used for VitastorFS" },
|
||||
{ "status", "Status" },
|
||||
{ "pg_count_fmt", "PGs" },
|
||||
{ "pg_minsize", "PG minsize" },
|
||||
{ "failure_domain", "Failure domain" },
|
||||
{ "root_node", "Root node" },
|
||||
{ "osd_tags_fmt", "OSD tags" },
|
||||
{ "primary_affinity_tags_fmt", "Primary affinity" },
|
||||
{ "block_size_fmt", "Block size" },
|
||||
{ "bitmap_granularity_fmt", "Bitmap granularity" },
|
||||
{ "immediate_commit", "Immediate commit" },
|
||||
{ "scrub_interval", "Scrub interval" },
|
||||
{ "inode_stats_fmt", "Per-inode stats" },
|
||||
{ "pg_stripe_size", "PG stripe size" },
|
||||
{ "max_osd_combinations", "Max OSD combinations" },
|
||||
{ "total_fmt", "Total" },
|
||||
{ "used_fmt", "Used" },
|
||||
{ "max_avail_fmt", "Available" },
|
||||
{ "used_pct", "Used%" },
|
||||
{ "eff_fmt", "Efficiency" },
|
||||
{ "osd_count", "OSD count" },
|
||||
{ "misplaced_fmt", "Misplaced" },
|
||||
{ "degraded_fmt", "Degraded" },
|
||||
{ "incomplete_fmt", "Incomplete" },
|
||||
{ "read_fmt", "Read" },
|
||||
{ "write_fmt", "Write" },
|
||||
{ "delete_fmt", "Delete" },
|
||||
};
|
||||
auto list = to_list();
|
||||
size_t title_len = 0;
|
||||
for (auto & item: list)
|
||||
{
|
||||
title_len = print_detail_title_len(item, cols, title_len);
|
||||
}
|
||||
for (auto & item: list)
|
||||
{
|
||||
if (result.text != "")
|
||||
result.text += "\n";
|
||||
result.text += print_detail(item, cols, title_len, parent->color);
|
||||
}
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
// Table output: name, scheme_name, pg_count, total, used, max_avail, used%, efficiency
|
||||
json11::Json::array cols;
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "name" },
|
||||
{ "title", "NAME" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "scheme_name" },
|
||||
{ "title", "SCHEME" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "status" },
|
||||
{ "title", "STATUS" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "pg_count_fmt" },
|
||||
{ "title", "PGS" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "total_fmt" },
|
||||
{ "title", "TOTAL" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "used_fmt" },
|
||||
{ "title", "USED" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "max_avail_fmt" },
|
||||
{ "title", "AVAILABLE" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "used_pct" },
|
||||
{ "title", "USED%" },
|
||||
});
|
||||
cols.push_back(json11::Json::object{
|
||||
{ "key", "eff_fmt" },
|
||||
{ "title", "EFFICIENCY" },
|
||||
});
|
||||
if (show_recovery)
|
||||
{
|
||||
cols.push_back(json11::Json::object{ { "key", "recovery_fmt" }, { "title", "RECOVERY" } });
|
||||
}
|
||||
if (show_stats)
|
||||
{
|
||||
cols.push_back(json11::Json::object{ { "key", "read_bw" }, { "title", "READ" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "read_iops" }, { "title", "IOPS" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "read_lat_f" }, { "title", "LAT" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "write_bw" }, { "title", "WRITE" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "write_iops" }, { "title", "IOPS" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "write_lat_f" }, { "title", "LAT" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "delete_bw" }, { "title", "DELETE" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "delete_iops" }, { "title", "IOPS" } });
|
||||
cols.push_back(json11::Json::object{ { "key", "delete_lat_f" }, { "title", "LAT" } });
|
||||
}
|
||||
result.data = to_list();
|
||||
result.text = print_table(result.data, cols, parent->color);
|
||||
state = 100;
|
||||
}
|
||||
};
|
||||
|
||||
size_t print_detail_title_len(json11::Json item, std::vector<std::pair<std::string, std::string>> names, size_t prev_len)
|
||||
{
|
||||
size_t title_len = prev_len;
|
||||
for (auto & kv: names)
|
||||
{
|
||||
if (!item[kv.first].is_null() && (!item[kv.first].is_string() || item[kv.first].string_value() != ""))
|
||||
{
|
||||
size_t len = utf8_length(kv.second);
|
||||
title_len = title_len < len ? len : title_len;
|
||||
}
|
||||
}
|
||||
return title_len;
|
||||
}
|
||||
|
||||
std::string print_detail(json11::Json item, std::vector<std::pair<std::string, std::string>> names, size_t title_len, bool use_esc)
|
||||
{
|
||||
std::string str;
|
||||
for (auto & kv: names)
|
||||
{
|
||||
if (!item[kv.first].is_null() && (!item[kv.first].is_string() || item[kv.first].string_value() != ""))
|
||||
{
|
||||
str += kv.second;
|
||||
str += ": ";
|
||||
size_t len = utf8_length(kv.second);
|
||||
for (int j = 0; j < title_len-len; j++)
|
||||
str += ' ';
|
||||
if (use_esc)
|
||||
str += "\033[1m";
|
||||
str += item[kv.first].as_string();
|
||||
if (use_esc)
|
||||
str += "\033[0m";
|
||||
str += "\n";
|
||||
}
|
||||
}
|
||||
return str;
|
||||
}
|
||||
|
||||
std::function<bool(cli_result_t &)> cli_tool_t::start_pool_ls(json11::Json cfg)
|
||||
{
|
||||
auto lister = new pool_lister_t();
|
||||
lister->parent = this;
|
||||
lister->show_recovery = cfg["show_recovery"].bool_value();
|
||||
lister->show_stats = cfg["long"].bool_value();
|
||||
lister->detailed = cfg["detail"].bool_value();
|
||||
lister->sort_field = cfg["sort"].string_value();
|
||||
if ((lister->sort_field == "osd_tags") ||
|
||||
(lister->sort_field == "primary_affinity_tags" ))
|
||||
lister->sort_field = lister->sort_field + "_fmt";
|
||||
lister->reverse = cfg["reverse"].bool_value();
|
||||
lister->max_count = cfg["count"].uint64_value();
|
||||
for (auto & item: cfg["names"].array_items())
|
||||
{
|
||||
lister->only_names.insert(item.string_value());
|
||||
}
|
||||
return [lister](cli_result_t & result)
|
||||
{
|
||||
lister->loop();
|
||||
if (lister->is_done())
|
||||
{
|
||||
result = lister->result;
|
||||
delete lister;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
||||
|
||||
std::string implode(const std::string & sep, json11::Json array)
|
||||
{
|
||||
if (array.is_number() || array.is_bool() || array.is_string())
|
||||
{
|
||||
return array.as_string();
|
||||
}
|
||||
std::string res;
|
||||
bool first = true;
|
||||
for (auto & item: array.array_items())
|
||||
{
|
||||
res += (first ? item.as_string() : sep+item.as_string());
|
||||
first = false;
|
||||
}
|
||||
return res;
|
||||
}
|
|
@ -0,0 +1,203 @@
|
|||
// Copyright (c) MIND Software LLC, 2023 (info@mindsw.io)
|
||||
// I accept Vitastor CLA: see CLA-en.md for details
|
||||
// Copyright (c) Vitaliy Filippov, 2024
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <ctype.h>
|
||||
#include "cli.h"
|
||||
#include "cli_pool_cfg.h"
|
||||
#include "cluster_client.h"
|
||||
#include "str_util.h"
|
||||
|
||||
struct pool_changer_t
|
||||
{
|
||||
cli_tool_t *parent;
|
||||
|
||||
// Required parameters (id/name)
|
||||
pool_id_t pool_id = 0;
|
||||
std::string pool_name;
|
||||
json11::Json::object cfg;
|
||||
json11::Json::object new_cfg;
|
||||
bool force = false;
|
||||
|
||||
json11::Json old_cfg;
|
||||
|
||||
int state = 0;
|
||||
cli_result_t result;
|
||||
|
||||
// Updated pools
|
||||
json11::Json new_pools;
|
||||
|
||||
// Expected pools mod revision
|
||||
uint64_t pools_mod_rev;
|
||||
|
||||
bool is_done() { return state == 100; }
|
||||
|
||||
void loop()
|
||||
{
|
||||
if (state == 1)
|
||||
goto resume_1;
|
||||
else if (state == 2)
|
||||
goto resume_2;
|
||||
pool_id = stoull_full(cfg["old_name"].string_value());
|
||||
if (!pool_id)
|
||||
{
|
||||
pool_name = cfg["old_name"].string_value();
|
||||
if (pool_name == "")
|
||||
{
|
||||
result = (cli_result_t){ .err = ENOENT, .text = "Pool ID or name is required to modify it" };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
}
|
||||
resume_0:
|
||||
// Get pools from etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
} }
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 1;
|
||||
resume_1:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
{
|
||||
// Parse received pools from etcd
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
|
||||
|
||||
// Get pool by name or ID
|
||||
old_cfg = json11::Json();
|
||||
if (pool_name != "")
|
||||
{
|
||||
for (auto & pce: kv.value.object_items())
|
||||
{
|
||||
if (pce.second["name"] == pool_name)
|
||||
{
|
||||
pool_id = stoull_full(pce.first);
|
||||
old_cfg = pce.second;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
pool_name = std::to_string(pool_id);
|
||||
old_cfg = kv.value[pool_name];
|
||||
}
|
||||
if (!old_cfg.is_object())
|
||||
{
|
||||
result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" does not exist" };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Update pool
|
||||
new_cfg = cfg;
|
||||
result.text = validate_pool_config(new_cfg, old_cfg, parent->cli->st_cli.global_block_size,
|
||||
parent->cli->st_cli.global_bitmap_granularity, force);
|
||||
if (result.text != "")
|
||||
{
|
||||
result.err = EINVAL;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
if (new_cfg.find("used_for_fs") != new_cfg.end() && !force)
|
||||
{
|
||||
// Check that pool doesn't have images
|
||||
auto img_it = parent->cli->st_cli.inode_config.lower_bound(INODE_WITH_POOL(pool_id, 0));
|
||||
if (img_it != parent->cli->st_cli.inode_config.end() && INODE_POOL(img_it->first) == pool_id &&
|
||||
img_it->second.name == new_cfg["used_for_fs"].string_value())
|
||||
{
|
||||
// Only allow metadata image to exist in the FS pool
|
||||
img_it++;
|
||||
}
|
||||
if (img_it != parent->cli->st_cli.inode_config.end() && INODE_POOL(img_it->first) == pool_id)
|
||||
{
|
||||
result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" has block images, delete them before using it for VitastorFS" };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Update pool
|
||||
auto pls = kv.value.object_items();
|
||||
pls[std::to_string(pool_id)] = new_cfg;
|
||||
new_pools = pls;
|
||||
|
||||
// Expected pools mod revision
|
||||
pools_mod_rev = kv.mod_revision;
|
||||
}
|
||||
// Update pools in etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "compare", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "target", "MOD" },
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "result", "LESS" },
|
||||
{ "mod_revision", pools_mod_rev+1 },
|
||||
}
|
||||
} },
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "value", base64_encode(new_pools.dump()) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 2;
|
||||
resume_2:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
if (!parent->etcd_result["succeeded"].bool_value())
|
||||
{
|
||||
// CAS failure - retry
|
||||
fprintf(stderr, "Warning: pool configuration was modified in the meantime by someone else\n");
|
||||
goto resume_0;
|
||||
}
|
||||
// Successfully updated pool
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Pool "+pool_name+" updated",
|
||||
.data = new_pools,
|
||||
};
|
||||
state = 100;
|
||||
}
|
||||
};
|
||||
|
||||
std::function<bool(cli_result_t &)> cli_tool_t::start_pool_modify(json11::Json cfg)
|
||||
{
|
||||
auto pool_changer = new pool_changer_t();
|
||||
pool_changer->parent = this;
|
||||
pool_changer->cfg = cfg.object_items();
|
||||
pool_changer->force = cfg["force"].bool_value();
|
||||
return [pool_changer](cli_result_t & result)
|
||||
{
|
||||
pool_changer->loop();
|
||||
if (pool_changer->is_done())
|
||||
{
|
||||
result = pool_changer->result;
|
||||
delete pool_changer;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
|
@ -0,0 +1,226 @@
|
|||
// Copyright (c) MIND Software LLC, 2023 (info@mindsw.io)
|
||||
// I accept Vitastor CLA: see CLA-en.md for details
|
||||
// Copyright (c) Vitaliy Filippov, 2024
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
#include <ctype.h>
|
||||
#include "cli.h"
|
||||
#include "cluster_client.h"
|
||||
#include "str_util.h"
|
||||
|
||||
struct pool_remover_t
|
||||
{
|
||||
cli_tool_t *parent;
|
||||
|
||||
// Required parameters (id/name)
|
||||
|
||||
pool_id_t pool_id = 0;
|
||||
std::string pool_name;
|
||||
|
||||
// Force removal
|
||||
bool force;
|
||||
|
||||
int state = 0;
|
||||
cli_result_t result;
|
||||
|
||||
// Is pool valid?
|
||||
bool pool_valid = false;
|
||||
|
||||
// Updated pools
|
||||
json11::Json new_pools;
|
||||
|
||||
// Expected pools mod revision
|
||||
uint64_t pools_mod_rev;
|
||||
|
||||
bool is_done() { return state == 100; }
|
||||
|
||||
void loop()
|
||||
{
|
||||
if (state == 1)
|
||||
goto resume_1;
|
||||
else if (state == 2)
|
||||
goto resume_2;
|
||||
else if (state == 3)
|
||||
goto resume_3;
|
||||
|
||||
// Pool name (or id) required
|
||||
if (!pool_id && pool_name == "")
|
||||
{
|
||||
result = (cli_result_t){ .err = EINVAL, .text = "Pool name or id must be given" };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate pool name/id
|
||||
|
||||
// Get pool id by name (if name given)
|
||||
if (pool_name != "")
|
||||
{
|
||||
for (auto & ic: parent->cli->st_cli.pool_config)
|
||||
{
|
||||
if (ic.second.name == pool_name)
|
||||
{
|
||||
pool_id = ic.first;
|
||||
pool_valid = 1;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Otherwise, check if given pool id is valid
|
||||
else
|
||||
{
|
||||
// Set pool name from id (for easier logging)
|
||||
pool_name = "id " + std::to_string(pool_id);
|
||||
|
||||
// Look-up pool id in pool_config
|
||||
if (parent->cli->st_cli.pool_config.find(pool_id) != parent->cli->st_cli.pool_config.end())
|
||||
{
|
||||
pool_valid = 1;
|
||||
}
|
||||
}
|
||||
|
||||
// Need a valid pool to proceed
|
||||
if (!pool_valid)
|
||||
{
|
||||
result = (cli_result_t){ .err = ENOENT, .text = "Pool "+pool_name+" does not exist" };
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Unless forced, check if pool has associated Images/Snapshots
|
||||
if (!force)
|
||||
{
|
||||
std::string images;
|
||||
|
||||
for (auto & ic: parent->cli->st_cli.inode_config)
|
||||
{
|
||||
if (pool_id && INODE_POOL(ic.second.num) != pool_id)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
images += ((images != "") ? ", " : "") + ic.second.name;
|
||||
}
|
||||
|
||||
if (images != "")
|
||||
{
|
||||
result = (cli_result_t){
|
||||
.err = ENOTEMPTY,
|
||||
.text =
|
||||
"Pool "+pool_name+" cannot be removed as it still has the following "
|
||||
"images/snapshots associated with it: "+images
|
||||
};
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Proceed to deleting the pool
|
||||
state = 1;
|
||||
do
|
||||
{
|
||||
resume_1:
|
||||
// Get pools from etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
} }
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 2;
|
||||
resume_2:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
{
|
||||
// Parse received pools from etcd
|
||||
auto kv = parent->cli->st_cli.parse_etcd_kv(parent->etcd_result["responses"][0]["response_range"]["kvs"][0]);
|
||||
|
||||
// Remove pool
|
||||
auto p = kv.value.object_items();
|
||||
if (p.erase(std::to_string(pool_id)) != 1)
|
||||
{
|
||||
result = (cli_result_t){
|
||||
.err = ENOENT,
|
||||
.text = "Failed to erase pool "+pool_name+" from: "+kv.value.string_value()
|
||||
};
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
|
||||
// Record updated pools
|
||||
new_pools = p;
|
||||
|
||||
// Expected pools mod revision
|
||||
pools_mod_rev = kv.mod_revision;
|
||||
}
|
||||
// Update pools in etcd
|
||||
parent->etcd_txn(json11::Json::object {
|
||||
{ "compare", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "target", "MOD" },
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "result", "LESS" },
|
||||
{ "mod_revision", pools_mod_rev+1 },
|
||||
}
|
||||
} },
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_put", json11::Json::object {
|
||||
{ "key", base64_encode(parent->cli->st_cli.etcd_prefix+"/config/pools") },
|
||||
{ "value", base64_encode(new_pools.dump()) },
|
||||
} },
|
||||
},
|
||||
} },
|
||||
});
|
||||
state = 3;
|
||||
resume_3:
|
||||
if (parent->waiting > 0)
|
||||
return;
|
||||
if (parent->etcd_err.err)
|
||||
{
|
||||
result = parent->etcd_err;
|
||||
state = 100;
|
||||
return;
|
||||
}
|
||||
} while (!parent->etcd_result["succeeded"].bool_value());
|
||||
|
||||
// Successfully deleted pool
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "Pool "+pool_name+" deleted",
|
||||
.data = new_pools
|
||||
};
|
||||
state = 100;
|
||||
}
|
||||
};
|
||||
|
||||
std::function<bool(cli_result_t &)> cli_tool_t::start_pool_rm(json11::Json cfg)
|
||||
{
|
||||
auto pool_remover = new pool_remover_t();
|
||||
pool_remover->parent = this;
|
||||
|
||||
pool_remover->pool_id = cfg["pool"].uint64_value();
|
||||
pool_remover->pool_name = pool_remover->pool_id ? "" : cfg["pool"].as_string();
|
||||
|
||||
pool_remover->force = !cfg["force"].is_null();
|
||||
|
||||
return [pool_remover](cli_result_t & result)
|
||||
{
|
||||
pool_remover->loop();
|
||||
if (pool_remover->is_done())
|
||||
{
|
||||
result = pool_remover->result;
|
||||
delete pool_remover;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
}
|
|
@ -53,6 +53,8 @@ struct snap_remover_t
|
|||
int use_cas = 1;
|
||||
// interval between fsyncs
|
||||
int fsync_interval = 128;
|
||||
// ignore deletion errors
|
||||
bool down_ok = false;
|
||||
|
||||
std::map<inode_t,int> sources;
|
||||
std::map<inode_t,uint64_t> inode_used;
|
||||
|
@ -245,6 +247,7 @@ resume_8:
|
|||
}
|
||||
state = 100;
|
||||
result = (cli_result_t){
|
||||
.err = 0,
|
||||
.text = "",
|
||||
.data = my_result(result.data),
|
||||
};
|
||||
|
@ -679,6 +682,7 @@ resume_100:
|
|||
{ "inode", inode },
|
||||
{ "pool", (uint64_t)INODE_POOL(inode) },
|
||||
{ "fsync-interval", fsync_interval },
|
||||
{ "down-ok", down_ok },
|
||||
});
|
||||
}
|
||||
};
|
||||
|
@ -690,6 +694,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm(json11::Json cfg)
|
|||
snap_remover->from_name = cfg["from"].string_value();
|
||||
snap_remover->to_name = cfg["to"].string_value();
|
||||
snap_remover->fsync_interval = cfg["fsync_interval"].uint64_value();
|
||||
snap_remover->down_ok = cfg["down_ok"].bool_value();
|
||||
if (!snap_remover->fsync_interval)
|
||||
snap_remover->fsync_interval = 128;
|
||||
if (!cfg["cas"].is_null())
|
||||
|
|
|
@ -25,6 +25,7 @@ struct rm_inode_t
|
|||
uint64_t inode = 0;
|
||||
pool_id_t pool_id = 0;
|
||||
uint64_t min_offset = 0;
|
||||
bool down_ok = false;
|
||||
|
||||
cli_tool_t *parent = NULL;
|
||||
inode_list_t *lister = NULL;
|
||||
|
@ -212,7 +213,9 @@ struct rm_inode_t
|
|||
}
|
||||
if (parent->progress && total_count > 0 && total_done*1000/total_count != total_prev_pct)
|
||||
{
|
||||
fprintf(stderr, "\rRemoved %ju/%ju objects, %ju more PGs to list...", total_done, total_count, pgs_to_list);
|
||||
fprintf(stderr, parent->color
|
||||
? "\rRemoved %ju/%ju objects, %ju more PGs to list..."
|
||||
: "Removed %ju/%ju objects, %ju more PGs to list...\n", total_done, total_count, pgs_to_list);
|
||||
total_prev_pct = total_done*1000/total_count;
|
||||
}
|
||||
if (lists_done && !lists.size())
|
||||
|
@ -221,17 +224,18 @@ struct rm_inode_t
|
|||
{
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
if (parent->progress && (total_done < total_count || inactive_osds.size() > 0 || error_count > 0))
|
||||
bool is_error = (total_done < total_count || inactive_osds.size() > 0 || error_count > 0);
|
||||
if (parent->progress && is_error)
|
||||
{
|
||||
fprintf(
|
||||
stderr, "Warning: Pool:%u,ID:%ju inode data may not have been fully removed.\n"
|
||||
" Use `vitastor-cli rm-data --pool %u --inode %ju` if you encounter it in listings.\n",
|
||||
"Use `vitastor-cli rm-data --pool %u --inode %ju` if you encounter it in listings.\n",
|
||||
pool_id, INODE_NO_POOL(inode), pool_id, INODE_NO_POOL(inode)
|
||||
);
|
||||
}
|
||||
result = (cli_result_t){
|
||||
.err = error_count > 0 ? EIO : 0,
|
||||
.text = error_count > 0 ? "Some blocks were not removed" : (
|
||||
.err = is_error && !down_ok ? EIO : 0,
|
||||
.text = is_error ? "Some blocks were not removed" : (
|
||||
"Done, inode "+std::to_string(INODE_NO_POOL(inode))+" from pool "+
|
||||
std::to_string(pool_id)+" removed"),
|
||||
.data = json11::Json::object {
|
||||
|
@ -280,6 +284,7 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_rm_data(json11::Json cfg)
|
|||
{
|
||||
remover->inode = (remover->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)) | (((uint64_t)remover->pool_id) << (64-POOL_ID_BITS));
|
||||
}
|
||||
remover->down_ok = cfg["down_ok"].bool_value();
|
||||
remover->pool_id = INODE_POOL(remover->inode);
|
||||
remover->min_offset = cfg["min_offset"].uint64_value();
|
||||
return [remover](cli_result_t & result)
|
||||
|
|
|
@ -6,7 +6,7 @@
|
|||
#include "cluster_client_impl.h"
|
||||
#include "http_client.h" // json_is_true
|
||||
|
||||
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config)
|
||||
cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config)
|
||||
{
|
||||
wb = new writeback_cache_t();
|
||||
|
||||
|
@ -238,7 +238,8 @@ void cluster_client_t::erase_op(cluster_op_t *op)
|
|||
// which may continue following SYNCs, but these SYNCs
|
||||
// should know about the changed buffer state
|
||||
// This is ugly but this is the way we do it
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
}
|
||||
if (!(flags & OP_IMMEDIATE_COMMIT) || enable_writeback)
|
||||
{
|
||||
|
@ -248,7 +249,8 @@ void cluster_client_t::erase_op(cluster_op_t *op)
|
|||
{
|
||||
// Call callback at the end to avoid inconsistencies in prev_wait
|
||||
// if the callback adds more operations itself
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
}
|
||||
if (flags & OP_FLUSH_BUFFER)
|
||||
{
|
||||
|
@ -265,7 +267,7 @@ void cluster_client_t::erase_op(cluster_op_t *op)
|
|||
}
|
||||
}
|
||||
|
||||
void cluster_client_t::continue_ops(bool up_retry)
|
||||
void cluster_client_t::continue_ops(int time_passed)
|
||||
{
|
||||
if (!pgs_loaded)
|
||||
{
|
||||
|
@ -277,22 +279,27 @@ void cluster_client_t::continue_ops(bool up_retry)
|
|||
// Attempt to reenter the function
|
||||
return;
|
||||
}
|
||||
int reset_duration = 0;
|
||||
restart:
|
||||
continuing_ops = 1;
|
||||
for (auto op = op_queue_head; op; )
|
||||
{
|
||||
cluster_op_t *next_op = op->next;
|
||||
if (!op->up_wait || up_retry)
|
||||
if (op->retry_after && time_passed)
|
||||
{
|
||||
op->up_wait = false;
|
||||
if (!op->prev_wait)
|
||||
op->retry_after = op->retry_after > time_passed ? op->retry_after-time_passed : 0;
|
||||
if (op->retry_after && (!reset_duration || op->retry_after < reset_duration))
|
||||
{
|
||||
reset_duration = op->retry_after;
|
||||
}
|
||||
}
|
||||
if (!op->retry_after && !op->prev_wait)
|
||||
{
|
||||
if (op->opcode == OSD_OP_SYNC)
|
||||
continue_sync(op);
|
||||
else
|
||||
continue_rw(op);
|
||||
}
|
||||
}
|
||||
op = next_op;
|
||||
if (continuing_ops == 2)
|
||||
{
|
||||
|
@ -300,6 +307,27 @@ restart:
|
|||
}
|
||||
}
|
||||
continuing_ops = 0;
|
||||
reset_retry_timer(reset_duration);
|
||||
}
|
||||
|
||||
void cluster_client_t::reset_retry_timer(int new_duration)
|
||||
{
|
||||
if (retry_timeout_duration && retry_timeout_duration <= new_duration || !new_duration)
|
||||
{
|
||||
return;
|
||||
}
|
||||
if (retry_timeout_id)
|
||||
{
|
||||
tfd->clear_timer(retry_timeout_id);
|
||||
}
|
||||
retry_timeout_duration = new_duration;
|
||||
retry_timeout_id = tfd->set_timer(retry_timeout_duration, false, [this](int)
|
||||
{
|
||||
int time_passed = retry_timeout_duration;
|
||||
retry_timeout_id = 0;
|
||||
retry_timeout_duration = 0;
|
||||
continue_ops(time_passed);
|
||||
});
|
||||
}
|
||||
|
||||
void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_config)
|
||||
|
@ -349,15 +377,25 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
|
|||
{
|
||||
client_max_writeback_iodepth = DEFAULT_CLIENT_MAX_WRITEBACK_IODEPTH;
|
||||
}
|
||||
// up_wait_retry_interval
|
||||
up_wait_retry_interval = config["up_wait_retry_interval"].uint64_value();
|
||||
if (!up_wait_retry_interval)
|
||||
// client_retry_interval
|
||||
client_retry_interval = config["client_retry_interval"].uint64_value();
|
||||
if (!client_retry_interval)
|
||||
{
|
||||
up_wait_retry_interval = 50;
|
||||
client_retry_interval = 50;
|
||||
}
|
||||
else if (up_wait_retry_interval < 10)
|
||||
else if (client_retry_interval < 10)
|
||||
{
|
||||
up_wait_retry_interval = 10;
|
||||
client_retry_interval = 10;
|
||||
}
|
||||
// client_eio_retry_interval
|
||||
client_eio_retry_interval = 1000;
|
||||
if (!config["client_eio_retry_interval"].is_null())
|
||||
{
|
||||
client_eio_retry_interval = config["client_eio_retry_interval"].uint64_value();
|
||||
if (client_eio_retry_interval && client_eio_retry_interval < 10)
|
||||
{
|
||||
client_eio_retry_interval = 10;
|
||||
}
|
||||
}
|
||||
// log_level
|
||||
log_level = config["log_level"].uint64_value();
|
||||
|
@ -512,7 +550,8 @@ void cluster_client_t::execute(cluster_op_t *op)
|
|||
op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_WRITE)
|
||||
{
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return;
|
||||
}
|
||||
if (!pgs_loaded)
|
||||
|
@ -534,7 +573,7 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
|
|||
return;
|
||||
}
|
||||
if (op->opcode == OSD_OP_WRITE && enable_writeback && !(op->flags & OP_FLUSH_BUFFER) &&
|
||||
!op->version /* FIXME no CAS writeback */)
|
||||
!op->version /* no CAS writeback */)
|
||||
{
|
||||
if (wb->writebacks_active >= client_max_writeback_iodepth)
|
||||
{
|
||||
|
@ -550,12 +589,13 @@ void cluster_client_t::execute_internal(cluster_op_t *op)
|
|||
wb->start_writebacks(this, 1);
|
||||
}
|
||||
op->retval = op->len;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return;
|
||||
}
|
||||
if (op->opcode == OSD_OP_WRITE && !(op->flags & OP_IMMEDIATE_COMMIT))
|
||||
{
|
||||
if (!(op->flags & OP_FLUSH_BUFFER))
|
||||
if (!(op->flags & OP_FLUSH_BUFFER) && !op->version /* no CAS write-repeat */)
|
||||
{
|
||||
wb->copy_write(op, CACHE_WRITTEN);
|
||||
}
|
||||
|
@ -619,7 +659,8 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
|
|||
if (!pool_id)
|
||||
{
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return false;
|
||||
}
|
||||
auto pool_it = st_cli.pool_config.find(pool_id);
|
||||
|
@ -627,15 +668,17 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
|
|||
{
|
||||
// Pools are loaded, but this one is unknown
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return false;
|
||||
}
|
||||
// Check alignment
|
||||
if (!op->len && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP || op->opcode == OSD_OP_WRITE) ||
|
||||
if (!op->len && (op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP || op->opcode == OSD_OP_WRITE) ||
|
||||
op->offset % pool_it->second.bitmap_granularity || op->len % pool_it->second.bitmap_granularity)
|
||||
{
|
||||
op->retval = -EINVAL;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return false;
|
||||
}
|
||||
if (pool_it->second.immediate_commit == IMMEDIATE_ALL)
|
||||
|
@ -648,7 +691,8 @@ bool cluster_client_t::check_rw(cluster_op_t *op)
|
|||
if (ino_it != st_cli.inode_config.end() && ino_it->second.readonly)
|
||||
{
|
||||
op->retval = -EROFS;
|
||||
std::function<void(cluster_op_t*)>(op->callback)(op);
|
||||
auto cb = std::move(op->callback);
|
||||
cb(op);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
@ -716,15 +760,8 @@ resume_1:
|
|||
// We'll need to retry again
|
||||
if (op->parts[i].flags & PART_RETRY)
|
||||
{
|
||||
op->up_wait = true;
|
||||
if (!retry_timeout_id)
|
||||
{
|
||||
retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
|
||||
{
|
||||
retry_timeout_id = 0;
|
||||
continue_ops(true);
|
||||
});
|
||||
}
|
||||
op->retry_after = client_retry_interval;
|
||||
reset_retry_timer(client_retry_interval);
|
||||
}
|
||||
op->state = 1;
|
||||
}
|
||||
|
@ -780,10 +817,9 @@ resume_2:
|
|||
return 1;
|
||||
}
|
||||
else if (op->retval != 0 && !(op->flags & OP_FLUSH_BUFFER) &&
|
||||
op->retval != -EPIPE && op->retval != -EIO && op->retval != -ENOSPC)
|
||||
op->retval != -EPIPE && (op->retval != -EIO || !client_eio_retry_interval) && op->retval != -ENOSPC)
|
||||
{
|
||||
// Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
|
||||
// FIXME: Add a parameter to allow to not wait for EIOs (incomplete or corrupted objects) to heal
|
||||
erase_op(op);
|
||||
return 1;
|
||||
}
|
||||
|
@ -1138,7 +1174,6 @@ static inline void mem_or(void *res, const void *r2, unsigned int len)
|
|||
void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
||||
{
|
||||
cluster_op_t *op = part->parent;
|
||||
op->inflight_count--;
|
||||
int expected = part->op.req.hdr.opcode == OSD_OP_SYNC ? 0 : part->op.req.rw.len;
|
||||
if (part->op.reply.hdr.retval != expected)
|
||||
{
|
||||
|
@ -1161,7 +1196,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
|||
);
|
||||
}
|
||||
}
|
||||
else
|
||||
else if (log_level > 0)
|
||||
{
|
||||
fprintf(
|
||||
stderr, "%s operation failed on OSD %ju: retval=%jd (expected %d)\n",
|
||||
|
@ -1171,16 +1206,17 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
|||
// All next things like timer, continue_sync/rw and stop_client may affect the operation again
|
||||
// So do all these things after modifying operation state, otherwise we may hit reenterability bugs
|
||||
// FIXME postpone such things to set_immediate here to avoid bugs
|
||||
// Mark op->up_wait = true to retry operation after a short pause (not immediately)
|
||||
op->up_wait = true;
|
||||
if (!retry_timeout_id)
|
||||
// Set op->retry_after to retry operation after a short pause (not immediately)
|
||||
if (!op->retry_after)
|
||||
{
|
||||
retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
|
||||
{
|
||||
retry_timeout_id = 0;
|
||||
continue_ops(true);
|
||||
});
|
||||
op->retry_after = op->retval == -EIO ? client_eio_retry_interval : client_retry_interval;
|
||||
}
|
||||
reset_retry_timer(op->retry_after);
|
||||
if (stop_fd >= 0)
|
||||
{
|
||||
msgr.stop_client(stop_fd);
|
||||
}
|
||||
op->inflight_count--;
|
||||
if (op->inflight_count == 0)
|
||||
{
|
||||
if (op->opcode == OSD_OP_SYNC)
|
||||
|
@ -1188,14 +1224,11 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
|||
else
|
||||
continue_rw(op);
|
||||
}
|
||||
if (stop_fd >= 0)
|
||||
{
|
||||
msgr.stop_client(stop_fd);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// OK
|
||||
op->inflight_count--;
|
||||
if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
|
||||
dirty_osds.insert(part->osd_num);
|
||||
part->flags |= PART_DONE;
|
||||
|
|
|
@ -59,7 +59,7 @@ protected:
|
|||
void *buf = NULL;
|
||||
cluster_op_t *orig_op = NULL;
|
||||
bool needs_reslice = false;
|
||||
bool up_wait = false;
|
||||
int retry_after = 0;
|
||||
int inflight_count = 0, done_count = 0;
|
||||
std::vector<cluster_op_part_t> parts;
|
||||
void *part_bitmaps = NULL;
|
||||
|
@ -92,9 +92,11 @@ class cluster_client_t
|
|||
uint64_t client_max_writeback_iodepth = 0;
|
||||
|
||||
int log_level = 0;
|
||||
int up_wait_retry_interval = 500; // ms
|
||||
int client_retry_interval = 50; // ms
|
||||
int client_eio_retry_interval = 1000; // ms
|
||||
|
||||
int retry_timeout_id = 0;
|
||||
int retry_timeout_duration = 0;
|
||||
std::vector<cluster_op_t*> offline_ops;
|
||||
cluster_op_t *op_queue_head = NULL, *op_queue_tail = NULL;
|
||||
writeback_cache_t *wb = NULL;
|
||||
|
@ -121,7 +123,7 @@ public:
|
|||
json11::Json::object cli_config, file_config, etcd_global_config;
|
||||
json11::Json::object config;
|
||||
|
||||
cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config);
|
||||
cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json config);
|
||||
~cluster_client_t();
|
||||
void execute(cluster_op_t *op);
|
||||
void execute_raw(osd_num_t osd_num, osd_op_t *op);
|
||||
|
@ -131,7 +133,7 @@ public:
|
|||
|
||||
bool get_immediate_commit(uint64_t inode);
|
||||
|
||||
void continue_ops(bool up_retry = false);
|
||||
void continue_ops(int time_passed = 0);
|
||||
inode_list_t *list_inode_start(inode_t inode,
|
||||
std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback);
|
||||
int list_pg_count(inode_list_t *lst);
|
||||
|
@ -152,6 +154,7 @@ protected:
|
|||
int continue_rw(cluster_op_t *op);
|
||||
bool check_rw(cluster_op_t *op);
|
||||
void slice_rw(cluster_op_t *op);
|
||||
void reset_retry_timer(int new_duration);
|
||||
bool try_send(cluster_op_t *op, int i);
|
||||
int continue_sync(cluster_op_t *op);
|
||||
void send_sync(cluster_op_t *op, cluster_op_part_t *part);
|
||||
|
|
|
@ -16,11 +16,6 @@
|
|||
void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
|
||||
{
|
||||
std::string device = cfg["device"].string_value();
|
||||
if (device == "")
|
||||
{
|
||||
fprintf(stderr, "Device path is missing\n");
|
||||
exit(1);
|
||||
}
|
||||
uint64_t data_block_size = parse_size(cfg["object_size"].string_value());
|
||||
uint64_t bitmap_granularity = parse_size(cfg["bitmap_granularity"].string_value());
|
||||
uint64_t journal_size = parse_size(cfg["journal_size"].string_value());
|
||||
|
@ -57,6 +52,11 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output)
|
|||
uint64_t orig_device_size = device_size;
|
||||
if (!device_size)
|
||||
{
|
||||
if (device == "")
|
||||
{
|
||||
fprintf(stderr, "Device path is missing\n");
|
||||
exit(1);
|
||||
}
|
||||
struct stat st;
|
||||
if (stat(device.c_str(), &st) < 0)
|
||||
{
|
||||
|
|
|
@ -60,7 +60,7 @@ static const char *help_text =
|
|||
" --block_size 128k/1M Set blockstore object size\n"
|
||||
" --bitmap_granularity 4k Set bitmap granularity\n"
|
||||
" --data_csum_type none Set data checksum type (crc32c or none)\n"
|
||||
" --csum_block_size 4k Set data checksum block size\n"
|
||||
" --csum_block_size 4k/32k Set data checksum block size (SSD/HDD default)\n"
|
||||
" --data_device_block 4k Override data device block size\n"
|
||||
" --meta_device_block 4k Override metadata device block size\n"
|
||||
" --journal_device_block 4k Override journal device block size\n"
|
||||
|
|
|
@ -132,9 +132,6 @@ void disk_tool_simple_offsets(json11::Json cfg, bool json_output);
|
|||
|
||||
uint64_t sscanf_json(const char *fmt, const json11::Json & str);
|
||||
void fromhexstr(const std::string & from, int bytes, uint8_t *to);
|
||||
std::string realpath_str(std::string path, bool nofail = true);
|
||||
std::string read_all_fd(int fd);
|
||||
std::string read_file(std::string file, bool allow_enoent = false);
|
||||
int disable_cache(std::string dev);
|
||||
std::string get_parent_device(std::string dev);
|
||||
bool json_is_true(const json11::Json & val);
|
||||
|
|
|
@ -111,6 +111,8 @@ int disk_tool_t::prepare_one(std::map<std::string, std::string> options, int is_
|
|||
options["block_size"] = "1M";
|
||||
if (is_hybrid && options["throttle_small_writes"] == "")
|
||||
options["throttle_small_writes"] = "1";
|
||||
if (!is_hybrid && options.find("data_csum_type") != options.end() && options.at("data_csum_type") != "")
|
||||
options["csum_block_size"] = "32k";
|
||||
}
|
||||
else if (!json_is_true(options["disable_data_fsync"]))
|
||||
{
|
||||
|
|
|
@ -42,36 +42,6 @@ void fromhexstr(const std::string & from, int bytes, uint8_t *to)
|
|||
}
|
||||
}
|
||||
|
||||
std::string realpath_str(std::string path, bool nofail)
|
||||
{
|
||||
char *p = realpath((char*)path.c_str(), NULL);
|
||||
if (!p)
|
||||
{
|
||||
fprintf(stderr, "Failed to resolve %s: %s\n", path.c_str(), strerror(errno));
|
||||
return nofail ? path : "";
|
||||
}
|
||||
std::string rp(p);
|
||||
free(p);
|
||||
return rp;
|
||||
}
|
||||
|
||||
std::string read_file(std::string file, bool allow_enoent)
|
||||
{
|
||||
std::string res;
|
||||
int fd = open(file.c_str(), O_RDONLY);
|
||||
if (fd < 0 || (res = read_all_fd(fd)) == "")
|
||||
{
|
||||
int err = errno;
|
||||
if (fd >= 0)
|
||||
close(fd);
|
||||
if (!allow_enoent || err != ENOENT)
|
||||
fprintf(stderr, "Can't read %s: %s\n", file.c_str(), strerror(err));
|
||||
return "";
|
||||
}
|
||||
close(fd);
|
||||
return res;
|
||||
}
|
||||
|
||||
// returns 1 = check error, 0 = write through, -1 = write back
|
||||
// (similar to 1 = warning, -1 = error, 0 = success in disable_cache)
|
||||
static int check_queue_cache(std::string dev, std::string parent_dev)
|
||||
|
|
|
@ -101,7 +101,7 @@ void epoll_manager_t::handle_uring_event()
|
|||
my_uring_prep_poll_add(sqe, epoll_fd, POLLIN);
|
||||
data->callback = [this](ring_data_t *data)
|
||||
{
|
||||
if (data->res < 0)
|
||||
if (data->res < 0 && data->res != -ECANCELED)
|
||||
{
|
||||
throw std::runtime_error(std::string("epoll failed: ") + strerror(-data->res));
|
||||
}
|
||||
|
|
|
@ -573,8 +573,7 @@ void etcd_state_client_t::load_global_config()
|
|||
{
|
||||
global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
|
||||
}
|
||||
global_immediate_commit = global_config["immediate_commit"].string_value() == "all"
|
||||
? IMMEDIATE_ALL : (global_config["immediate_commit"].string_value() == "small" ? IMMEDIATE_SMALL : IMMEDIATE_NONE);
|
||||
global_immediate_commit = parse_immediate_commit(global_config["immediate_commit"].string_value());
|
||||
on_load_config_hook(global_config);
|
||||
});
|
||||
}
|
||||
|
@ -782,13 +781,8 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
|
|||
// Failure Domain
|
||||
pc.failure_domain = pool_item.second["failure_domain"].string_value();
|
||||
// Coding Scheme
|
||||
if (pool_item.second["scheme"] == "replicated")
|
||||
pc.scheme = POOL_SCHEME_REPLICATED;
|
||||
else if (pool_item.second["scheme"] == "xor")
|
||||
pc.scheme = POOL_SCHEME_XOR;
|
||||
else if (pool_item.second["scheme"] == "ec" || pool_item.second["scheme"] == "jerasure")
|
||||
pc.scheme = POOL_SCHEME_EC;
|
||||
else
|
||||
pc.scheme = parse_scheme(pool_item.second["scheme"].string_value());
|
||||
if (!pc.scheme)
|
||||
{
|
||||
fprintf(stderr, "Pool %u has invalid coding scheme (one of \"xor\", \"replicated\", \"ec\" or \"jerasure\" required), skipping pool\n", pool_id);
|
||||
continue;
|
||||
|
@ -869,11 +863,11 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
|
|||
pc.scrub_interval = parse_time(pool_item.second["scrub_interval"].string_value());
|
||||
if (!pc.scrub_interval)
|
||||
pc.scrub_interval = 0;
|
||||
// Mark pool as VitastorFS pool (disable per-inode stats and block volume creation)
|
||||
pc.used_for_fs = pool_item.second["used_for_fs"].as_string();
|
||||
// Immediate Commit Mode
|
||||
pc.immediate_commit = pool_item.second["immediate_commit"].is_string()
|
||||
? (pool_item.second["immediate_commit"].string_value() == "all"
|
||||
? IMMEDIATE_ALL : (pool_item.second["immediate_commit"].string_value() == "small"
|
||||
? IMMEDIATE_SMALL : IMMEDIATE_NONE))
|
||||
? parse_immediate_commit(pool_item.second["immediate_commit"].string_value())
|
||||
: global_immediate_commit;
|
||||
// PG Stripe Size
|
||||
pc.pg_stripe_size = pool_item.second["pg_stripe_size"].uint64_value();
|
||||
|
@ -1167,6 +1161,23 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
|
|||
}
|
||||
}
|
||||
|
||||
uint32_t etcd_state_client_t::parse_immediate_commit(const std::string & immediate_commit_str)
|
||||
{
|
||||
return immediate_commit_str == "all" ? IMMEDIATE_ALL :
|
||||
(immediate_commit_str == "small" ? IMMEDIATE_SMALL : IMMEDIATE_NONE);
|
||||
}
|
||||
|
||||
uint32_t etcd_state_client_t::parse_scheme(const std::string & scheme)
|
||||
{
|
||||
if (scheme == "replicated")
|
||||
return POOL_SCHEME_REPLICATED;
|
||||
else if (scheme == "xor")
|
||||
return POOL_SCHEME_XOR;
|
||||
else if (scheme == "ec" || scheme == "jerasure")
|
||||
return POOL_SCHEME_EC;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void etcd_state_client_t::insert_inode_config(const inode_config_t & cfg)
|
||||
{
|
||||
this->inode_config[cfg.num] = cfg;
|
||||
|
|
|
@ -60,6 +60,7 @@ struct pool_config_t
|
|||
uint64_t pg_stripe_size;
|
||||
std::map<pg_num_t, pg_config_t> pg_config;
|
||||
uint64_t scrub_interval;
|
||||
std::string used_for_fs;
|
||||
};
|
||||
|
||||
struct inode_config_t
|
||||
|
@ -151,4 +152,7 @@ public:
|
|||
void close_watch(inode_watch_t* watch);
|
||||
int address_count();
|
||||
~etcd_state_client_t();
|
||||
|
||||
static uint32_t parse_immediate_commit(const std::string & immediate_commit_str);
|
||||
static uint32_t parse_scheme(const std::string & scheme_str);
|
||||
};
|
||||
|
|
|
@ -0,0 +1,673 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
//
|
||||
// Vitastor shared key/value database test CLI
|
||||
|
||||
#define _XOPEN_SOURCE
|
||||
#include <limits.h>
|
||||
|
||||
#include <netinet/tcp.h>
|
||||
#include <sys/epoll.h>
|
||||
#include <unistd.h>
|
||||
#include <fcntl.h>
|
||||
//#include <signal.h>
|
||||
|
||||
#include "epoll_manager.h"
|
||||
#include "str_util.h"
|
||||
#include "kv_db.h"
|
||||
|
||||
const char *exe_name = NULL;
|
||||
|
||||
class kv_cli_t
|
||||
{
|
||||
public:
|
||||
json11::Json::object cfg;
|
||||
std::vector<std::string> cli_cmd;
|
||||
|
||||
kv_dbw_t *db = NULL;
|
||||
ring_loop_t *ringloop = NULL;
|
||||
epoll_manager_t *epmgr = NULL;
|
||||
cluster_client_t *cli = NULL;
|
||||
int load_parallelism = 16;
|
||||
bool opened = false;
|
||||
bool interactive = false, is_file = false;
|
||||
int in_progress = 0;
|
||||
char *cur_cmd = NULL;
|
||||
int cur_cmd_size = 0, cur_cmd_alloc = 0;
|
||||
bool finished = false, eof = false;
|
||||
|
||||
std::function<void(int)> load_cb;
|
||||
bool loading_json = false, in_loadjson = false;
|
||||
int load_state = 0;
|
||||
std::string load_key;
|
||||
|
||||
~kv_cli_t();
|
||||
|
||||
void parse_args(int narg, const char *args[]);
|
||||
void run();
|
||||
void read_cmd();
|
||||
void next_cmd();
|
||||
std::vector<std::string> parse_cmd(const std::string & cmdstr);
|
||||
void handle_cmd(const std::vector<std::string> & cmd, std::function<void(int)> cb);
|
||||
void loadjson();
|
||||
};
|
||||
|
||||
kv_cli_t::~kv_cli_t()
|
||||
{
|
||||
if (cur_cmd)
|
||||
{
|
||||
free(cur_cmd);
|
||||
cur_cmd = NULL;
|
||||
}
|
||||
cur_cmd_alloc = 0;
|
||||
if (db)
|
||||
delete db;
|
||||
if (cli)
|
||||
{
|
||||
cli->flush();
|
||||
delete cli;
|
||||
}
|
||||
if (epmgr)
|
||||
delete epmgr;
|
||||
if (ringloop)
|
||||
delete ringloop;
|
||||
}
|
||||
|
||||
void kv_cli_t::parse_args(int narg, const char *args[])
|
||||
{
|
||||
bool db = false;
|
||||
for (int i = 1; i < narg; i++)
|
||||
{
|
||||
if (!strcmp(args[i], "-h") || !strcmp(args[i], "--help"))
|
||||
{
|
||||
printf(
|
||||
"Vitastor Key/Value CLI\n"
|
||||
"(c) Vitaliy Filippov, 2023+ (VNPL-1.1)\n"
|
||||
"\n"
|
||||
"USAGE: %s [OPTIONS] [<IMAGE> [<COMMAND>]]\n"
|
||||
"\n"
|
||||
"COMMANDS:\n"
|
||||
" get <key>\n"
|
||||
" set <key> <value>\n"
|
||||
" del <key>\n"
|
||||
" list [<start> [end]]\n"
|
||||
" dump [<start> [end]]\n"
|
||||
" dumpjson [<start> [end]]\n"
|
||||
" loadjson\n"
|
||||
"\n"
|
||||
"<IMAGE> should be the name of Vitastor image with the DB.\n"
|
||||
"Without <COMMAND>, you get an interactive DB shell.\n"
|
||||
"\n"
|
||||
"OPTIONS:\n"
|
||||
" --kv_block_size 4k\n"
|
||||
" Key-value B-Tree block size\n"
|
||||
" --kv_memory_limit 128M\n"
|
||||
" Maximum memory to use for vitastor-kv index cache\n"
|
||||
" --kv_allocate_blocks 4\n"
|
||||
" Number of PG blocks used for new tree block allocation in parallel\n"
|
||||
" --kv_evict_max_misses 10\n"
|
||||
" Eviction algorithm parameter: retry eviction from another random spot\n"
|
||||
" if this number of keys is used currently or was used recently\n"
|
||||
" --kv_evict_attempts_per_level 3\n"
|
||||
" Retry eviction at most this number of times per tree level, starting\n"
|
||||
" with bottom-most levels\n"
|
||||
" --kv_evict_unused_age 1000\n"
|
||||
" Evict only keys unused during this number of last operations\n"
|
||||
" --kv_log_level 1\n"
|
||||
" Log level. 0 = errors, 1 = warnings, 10 = trace operations\n"
|
||||
,
|
||||
exe_name
|
||||
);
|
||||
exit(0);
|
||||
}
|
||||
else if (args[i][0] == '-' && args[i][1] == '-')
|
||||
{
|
||||
const char *opt = args[i]+2;
|
||||
cfg[opt] = !strcmp(opt, "json") || i == narg-1 ? "1" : args[++i];
|
||||
}
|
||||
else if (!db)
|
||||
{
|
||||
cfg["db"] = args[i];
|
||||
db = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
cli_cmd.push_back(args[i]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void kv_cli_t::run()
|
||||
{
|
||||
// Create client
|
||||
ringloop = new ring_loop_t(512);
|
||||
epmgr = new epoll_manager_t(ringloop);
|
||||
cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
|
||||
db = new kv_dbw_t(cli);
|
||||
// Load image metadata
|
||||
while (!cli->is_ready())
|
||||
{
|
||||
ringloop->loop();
|
||||
if (cli->is_ready())
|
||||
break;
|
||||
ringloop->wait();
|
||||
}
|
||||
// Open if DB is set in options
|
||||
if (cfg.find("db") != cfg.end())
|
||||
{
|
||||
bool done = false;
|
||||
handle_cmd({ "open", cfg.at("db").string_value() }, [&done](int res) { if (res != 0) exit(1); done = true; });
|
||||
while (!done)
|
||||
{
|
||||
ringloop->loop();
|
||||
if (done)
|
||||
break;
|
||||
ringloop->wait();
|
||||
}
|
||||
}
|
||||
// Run single command from CLI
|
||||
if (cli_cmd.size())
|
||||
{
|
||||
bool done = false;
|
||||
handle_cmd(cli_cmd, [&done](int res) { if (res != 0) exit(1); done = true; });
|
||||
while (!done)
|
||||
{
|
||||
ringloop->loop();
|
||||
if (done)
|
||||
break;
|
||||
ringloop->wait();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Run interactive shell
|
||||
fcntl(0, F_SETFL, fcntl(0, F_GETFL, 0) | O_NONBLOCK);
|
||||
try
|
||||
{
|
||||
epmgr->tfd->set_fd_handler(0, false, [this](int fd, int events)
|
||||
{
|
||||
if (events & EPOLLIN)
|
||||
{
|
||||
read_cmd();
|
||||
}
|
||||
if (events & EPOLLRDHUP)
|
||||
{
|
||||
epmgr->tfd->set_fd_handler(0, false, NULL);
|
||||
finished = true;
|
||||
}
|
||||
});
|
||||
interactive = isatty(0);
|
||||
if (interactive)
|
||||
printf("> ");
|
||||
}
|
||||
catch (std::exception & e)
|
||||
{
|
||||
// Can't add to epoll, STDIN is probably a file
|
||||
is_file = true;
|
||||
read_cmd();
|
||||
}
|
||||
while (!finished)
|
||||
{
|
||||
ringloop->loop();
|
||||
if (!finished)
|
||||
ringloop->wait();
|
||||
}
|
||||
}
|
||||
// Destroy the client
|
||||
delete db;
|
||||
db = NULL;
|
||||
cli->flush();
|
||||
delete cli;
|
||||
delete epmgr;
|
||||
delete ringloop;
|
||||
cli = NULL;
|
||||
epmgr = NULL;
|
||||
ringloop = NULL;
|
||||
}
|
||||
|
||||
void kv_cli_t::read_cmd()
|
||||
{
|
||||
if (!cur_cmd_alloc)
|
||||
{
|
||||
cur_cmd_alloc = 65536;
|
||||
cur_cmd = (char*)malloc_or_die(cur_cmd_alloc);
|
||||
}
|
||||
while (cur_cmd_size < cur_cmd_alloc)
|
||||
{
|
||||
int r = read(0, cur_cmd+cur_cmd_size, cur_cmd_alloc-cur_cmd_size);
|
||||
if (r < 0 && errno != EAGAIN)
|
||||
fprintf(stderr, "Error reading from stdin: %s\n", strerror(errno));
|
||||
if (r > 0)
|
||||
cur_cmd_size += r;
|
||||
if (r == 0)
|
||||
eof = true;
|
||||
if (r <= 0)
|
||||
break;
|
||||
}
|
||||
next_cmd();
|
||||
}
|
||||
|
||||
void kv_cli_t::next_cmd()
|
||||
{
|
||||
if (loading_json)
|
||||
{
|
||||
loadjson();
|
||||
return;
|
||||
}
|
||||
if (in_progress > 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
int pos = 0;
|
||||
for (; pos < cur_cmd_size; pos++)
|
||||
{
|
||||
if (cur_cmd[pos] == '\n' || cur_cmd[pos] == '\r')
|
||||
{
|
||||
auto cmd = trim(std::string(cur_cmd, pos));
|
||||
pos++;
|
||||
memmove(cur_cmd, cur_cmd+pos, cur_cmd_size-pos);
|
||||
cur_cmd_size -= pos;
|
||||
in_progress++;
|
||||
handle_cmd(parse_cmd(cmd), [this](int res)
|
||||
{
|
||||
in_progress--;
|
||||
if (interactive)
|
||||
printf("> ");
|
||||
next_cmd();
|
||||
if (!in_progress)
|
||||
read_cmd();
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (eof && !in_progress)
|
||||
{
|
||||
finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
struct kv_cli_list_t
|
||||
{
|
||||
kv_dbw_t *db = NULL;
|
||||
void *handle = NULL;
|
||||
int format = 0;
|
||||
int n = 0;
|
||||
std::function<void(int)> cb;
|
||||
};
|
||||
|
||||
std::vector<std::string> kv_cli_t::parse_cmd(const std::string & str)
|
||||
{
|
||||
std::vector<std::string> res;
|
||||
size_t pos = 0;
|
||||
auto cmd = scan_escaped(str, pos);
|
||||
if (cmd.empty())
|
||||
return res;
|
||||
res.push_back(cmd);
|
||||
int max_args = (cmd == "set" || cmd == "config" ||
|
||||
cmd == "list" || cmd == "dump" || cmd == "dumpjson" ? 3 :
|
||||
(cmd == "open" || cmd == "get" || cmd == "del" ? 2 : 1));
|
||||
while (pos < str.size() && res.size() < max_args)
|
||||
{
|
||||
if (res.size() == max_args-1)
|
||||
{
|
||||
// Allow unquoted last argument
|
||||
pos = str.find_first_not_of(" \t\r\n", pos);
|
||||
if (pos == std::string::npos)
|
||||
break;
|
||||
if (str[pos] != '"' && str[pos] != '\'')
|
||||
{
|
||||
res.push_back(trim(str.substr(pos)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
auto arg = scan_escaped(str, pos);
|
||||
if (arg.size())
|
||||
res.push_back(arg);
|
||||
}
|
||||
return res;
|
||||
}
|
||||
|
||||
void kv_cli_t::loadjson()
|
||||
{
|
||||
// simple streaming json parser
|
||||
if (in_progress >= load_parallelism || in_loadjson)
|
||||
{
|
||||
return;
|
||||
}
|
||||
in_loadjson = true;
|
||||
if (load_state == 5)
|
||||
{
|
||||
st_5:
|
||||
if (!in_progress)
|
||||
{
|
||||
loading_json = false;
|
||||
auto cb = std::move(load_cb);
|
||||
cb(0);
|
||||
}
|
||||
in_loadjson = false;
|
||||
return;
|
||||
}
|
||||
do
|
||||
{
|
||||
read_cmd();
|
||||
size_t pos = 0;
|
||||
while (true)
|
||||
{
|
||||
while (pos < cur_cmd_size && is_white(cur_cmd[pos]))
|
||||
{
|
||||
pos++;
|
||||
}
|
||||
if (pos >= cur_cmd_size)
|
||||
{
|
||||
break;
|
||||
}
|
||||
if (load_state == 0 || load_state == 2)
|
||||
{
|
||||
char expected = "{ :"[load_state];
|
||||
if (cur_cmd[pos] != expected)
|
||||
{
|
||||
fprintf(stderr, "Unexpected %c, expected %c\n", cur_cmd[pos], expected);
|
||||
exit(1);
|
||||
}
|
||||
pos++;
|
||||
load_state++;
|
||||
}
|
||||
else if (load_state == 1 || load_state == 3)
|
||||
{
|
||||
if (cur_cmd[pos] != '"')
|
||||
{
|
||||
fprintf(stderr, "Unexpected %c, expected \"\n", cur_cmd[pos]);
|
||||
exit(1);
|
||||
}
|
||||
size_t prev = pos;
|
||||
auto str = scan_escaped(cur_cmd, cur_cmd_size, pos, false);
|
||||
if (pos == prev)
|
||||
{
|
||||
break;
|
||||
}
|
||||
load_state++;
|
||||
if (load_state == 2)
|
||||
{
|
||||
load_key = str;
|
||||
}
|
||||
else
|
||||
{
|
||||
in_progress++;
|
||||
handle_cmd({ "set", load_key, str }, [this](int res)
|
||||
{
|
||||
in_progress--;
|
||||
next_cmd();
|
||||
});
|
||||
if (in_progress >= load_parallelism)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
else if (load_state == 4)
|
||||
{
|
||||
if (cur_cmd[pos] == ',')
|
||||
{
|
||||
pos++;
|
||||
load_state = 1;
|
||||
}
|
||||
else if (cur_cmd[pos] == '}')
|
||||
{
|
||||
pos++;
|
||||
load_state = 5;
|
||||
goto st_5;
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(stderr, "Unexpected %c, expected , or }\n", cur_cmd[pos]);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pos < cur_cmd_size)
|
||||
{
|
||||
memmove(cur_cmd, cur_cmd+pos, cur_cmd_size-pos);
|
||||
}
|
||||
cur_cmd_size -= pos;
|
||||
} while (loading_json && is_file);
|
||||
in_loadjson = false;
|
||||
}
|
||||
|
||||
void kv_cli_t::handle_cmd(const std::vector<std::string> & cmd, std::function<void(int)> cb)
|
||||
{
|
||||
if (!cmd.size())
|
||||
{
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
auto & opname = cmd[0];
|
||||
if (!opened && opname != "open" && opname != "config" && opname != "quit" && opname != "q")
|
||||
{
|
||||
fprintf(stderr, "Error: database not opened\n");
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
if (opname == "open")
|
||||
{
|
||||
auto name = cmd.size() > 1 ? cmd[1] : "";
|
||||
uint64_t pool_id = 0;
|
||||
inode_t inode_id = 0;
|
||||
int scanned = sscanf(name.c_str(), "%lu %lu", &pool_id, &inode_id);
|
||||
if (scanned < 2 || !pool_id || !inode_id)
|
||||
{
|
||||
inode_id = 0;
|
||||
name = trim(name);
|
||||
for (auto & ic: cli->st_cli.inode_config)
|
||||
{
|
||||
if (ic.second.name == name)
|
||||
{
|
||||
inode_id = ic.first;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!inode_id)
|
||||
{
|
||||
fprintf(stderr, "Usage: open <image> OR open <pool_id> <inode_id>\n");
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
}
|
||||
else
|
||||
inode_id = INODE_WITH_POOL(pool_id, inode_id);
|
||||
db->open(inode_id, cfg, [=](int res)
|
||||
{
|
||||
if (res < 0)
|
||||
{
|
||||
fprintf(stderr, "Error opening index: %s (code %d)\n", strerror(-res), res);
|
||||
}
|
||||
else
|
||||
{
|
||||
opened = true;
|
||||
fprintf(interactive ? stdout : stderr, "Index opened. Current size: %lu bytes\n", db->get_size());
|
||||
}
|
||||
cb(res);
|
||||
});
|
||||
}
|
||||
else if (opname == "config")
|
||||
{
|
||||
if (cmd.size() < 3)
|
||||
{
|
||||
fprintf(stderr, "Usage: config <property> <value>\n");
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
auto & key = cmd[1];
|
||||
auto & value = cmd[2];
|
||||
if (key != "kv_memory_limit" &&
|
||||
key != "kv_allocate_blocks" &&
|
||||
key != "kv_evict_max_misses" &&
|
||||
key != "kv_evict_attempts_per_level" &&
|
||||
key != "kv_evict_unused_age" &&
|
||||
key != "kv_log_level" &&
|
||||
key != "kv_block_size")
|
||||
{
|
||||
fprintf(
|
||||
stderr, "Allowed properties: kv_block_size, kv_memory_limit, kv_allocate_blocks,"
|
||||
" kv_evict_max_misses, kv_evict_attempts_per_level, kv_evict_unused_age, kv_log_level\n"
|
||||
);
|
||||
cb(-EINVAL);
|
||||
}
|
||||
else if (key == "kv_block_size")
|
||||
{
|
||||
if (opened)
|
||||
{
|
||||
fprintf(stderr, "kv_block_size can't be set after opening DB\n");
|
||||
cb(-EINVAL);
|
||||
}
|
||||
else
|
||||
{
|
||||
cfg[key] = value;
|
||||
cb(0);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
cfg[key] = value;
|
||||
db->set_config(cfg);
|
||||
cb(0);
|
||||
}
|
||||
}
|
||||
else if (opname == "get" || opname == "set" || opname == "del")
|
||||
{
|
||||
if (opname == "get" || opname == "del")
|
||||
{
|
||||
if (cmd.size() < 2)
|
||||
{
|
||||
fprintf(stderr, "Usage: %s <key>\n", opname.c_str());
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
auto & key = cmd[1];
|
||||
if (opname == "get")
|
||||
{
|
||||
db->get(key, [this, cb](int res, const std::string & value)
|
||||
{
|
||||
if (res < 0)
|
||||
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
|
||||
else
|
||||
{
|
||||
if (write(1, value.c_str(), value.size()) < 0 || write(1, "\n", 1) < 0)
|
||||
exit(1);
|
||||
}
|
||||
cb(res);
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
db->del(key, [this, cb](int res)
|
||||
{
|
||||
if (res < 0)
|
||||
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
|
||||
else
|
||||
fprintf(interactive ? stdout : stderr, "OK\n");
|
||||
cb(res);
|
||||
});
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if (cmd.size() < 3)
|
||||
{
|
||||
fprintf(stderr, "Usage: set <key> <value>\n");
|
||||
cb(-EINVAL);
|
||||
return;
|
||||
}
|
||||
auto & key = cmd[1];
|
||||
auto & value = cmd[2];
|
||||
db->set(key, value, [this, cb, l = loading_json](int res)
|
||||
{
|
||||
if (res < 0)
|
||||
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
|
||||
else if (!l)
|
||||
fprintf(interactive ? stdout : stderr, "OK\n");
|
||||
cb(res);
|
||||
});
|
||||
}
|
||||
}
|
||||
else if (opname == "list" || opname == "dump" || opname == "dumpjson")
|
||||
{
|
||||
kv_cli_list_t *lst = new kv_cli_list_t;
|
||||
std::string start = cmd.size() >= 2 ? cmd[1] : "";
|
||||
std::string end = cmd.size() >= 3 ? cmd[2] : "";
|
||||
lst->handle = db->list_start(start);
|
||||
lst->db = db;
|
||||
lst->format = opname == "dump" ? 1 : (opname == "dumpjson" ? 2 : 0);
|
||||
lst->cb = std::move(cb);
|
||||
db->list_next(lst->handle, [lst](int res, const std::string & key, const std::string & value)
|
||||
{
|
||||
if (res < 0)
|
||||
{
|
||||
if (res != -ENOENT)
|
||||
{
|
||||
fprintf(stderr, "Error: %s (code %d)\n", strerror(-res), res);
|
||||
}
|
||||
if (lst->format == 2)
|
||||
printf("\n}\n");
|
||||
lst->db->list_close(lst->handle);
|
||||
lst->cb(res == -ENOENT ? 0 : res);
|
||||
delete lst;
|
||||
}
|
||||
else
|
||||
{
|
||||
if (lst->format == 2)
|
||||
printf(lst->n ? ",\n %s: %s" : "{\n %s: %s", addslashes(key).c_str(), addslashes(value).c_str());
|
||||
else if (lst->format == 1)
|
||||
printf("set %s %s\n", auto_addslashes(key).c_str(), value.c_str());
|
||||
else
|
||||
printf("%s = %s\n", key.c_str(), value.c_str());
|
||||
lst->n++;
|
||||
lst->db->list_next(lst->handle, NULL);
|
||||
}
|
||||
});
|
||||
}
|
||||
else if (opname == "loadjson")
|
||||
{
|
||||
loading_json = true;
|
||||
load_state = 0;
|
||||
load_cb = cb;
|
||||
loadjson();
|
||||
}
|
||||
else if (opname == "close")
|
||||
{
|
||||
db->close([=]()
|
||||
{
|
||||
fprintf(interactive ? stdout : stderr, "Index closed\n");
|
||||
opened = false;
|
||||
cb(0);
|
||||
});
|
||||
}
|
||||
else if (opname == "quit" || opname == "q")
|
||||
{
|
||||
::close(0);
|
||||
finished = true;
|
||||
}
|
||||
else
|
||||
{
|
||||
fprintf(
|
||||
stderr, "Unknown operation: %s. Supported operations:\n"
|
||||
"open <image>\nopen <pool_id> <inode_id>\n"
|
||||
"config <property> <value>\n"
|
||||
"get <key>\nset <key> <value>\ndel <key>\n"
|
||||
"list [<start> [end]]\ndump [<start> [end]]\ndumpjson [<start> [end]]\nloadjson\n"
|
||||
"close\nquit\n", opname.c_str()
|
||||
);
|
||||
cb(-EINVAL);
|
||||
}
|
||||
}
|
||||
|
||||
int main(int narg, const char *args[])
|
||||
{
|
||||
setvbuf(stdout, NULL, _IONBF, 0);
|
||||
setvbuf(stderr, NULL, _IONBF, 0);
|
||||
exe_name = args[0];
|
||||
kv_cli_t *p = new kv_cli_t();
|
||||
p->parse_args(narg, args);
|
||||
p->run();
|
||||
delete p;
|
||||
return 0;
|
||||
}
|
File diff suppressed because it is too large
Load Diff
|
@ -0,0 +1,36 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
//
|
||||
// Vitastor shared key/value database
|
||||
// Parallel optimistic B-Tree O:-)
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "cluster_client.h"
|
||||
|
||||
struct kv_db_t;
|
||||
|
||||
struct kv_dbw_t
|
||||
{
|
||||
kv_dbw_t(cluster_client_t *cli);
|
||||
~kv_dbw_t();
|
||||
|
||||
void open(inode_t inode_id, json11::Json cfg, std::function<void(int)> cb);
|
||||
void set_config(json11::Json cfg);
|
||||
void close(std::function<void()> cb);
|
||||
|
||||
uint64_t get_size();
|
||||
|
||||
void get(const std::string & key, std::function<void(int res, const std::string & value)> cb,
|
||||
bool allow_old_cached = false);
|
||||
void set(const std::string & key, const std::string & value, std::function<void(int res)> cb,
|
||||
std::function<bool(int res, const std::string & value)> cas_compare = NULL);
|
||||
void del(const std::string & key, std::function<void(int res)> cb,
|
||||
std::function<bool(int res, const std::string & value)> cas_compare = NULL);
|
||||
|
||||
void* list_start(const std::string & start);
|
||||
void list_next(void *handle, std::function<void(int res, const std::string & key, const std::string & value)> cb);
|
||||
void list_close(void *handle);
|
||||
|
||||
kv_db_t *db;
|
||||
};
|
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue