Wait for RDMA-CM EVENT_ESTABLISHED after rdma_accept(), handle rdma_accept() before acking the event

New features:

- Support separate OSD cluster network - [osd_cluster_network](https://vitastor.io/docs/config/network.html#osd_cluster_network)
  and, in general, multiple OSD networks, including RDMA
- Add an alternative RDMA implementation via RDMA-CM - [use_rdmacm](https://vitastor.io/docs/config/network.html#use_rdmacm),
  required for iWARP and, maybe, for some IB setups (but not for RoCE)
- Change default PG behaviour to wait for all "up" OSDs to be connected before starting it.
  The old behaviour may be returned by enabling a new [allow_net_split](https://vitastor.io/docs/config/osd.html#allow_net_split)
  option.
- Add a patch for QEMU 9.2

Bug fixes:

- Fix incorrect "has_xxx" PG state names in ls-pgs
- Fix possible QEMU crashes after detaching of Vitastor disks (and update all QEMU builds in Vitastor repos)
- Fix clients sometimes spamming OSDs with infinite reconnections when some PGs are offline
- Fall back to TCP on RDMA connection failures
- Add missing logging of RDMA ibv_modify_qp() errors
- Add a minimum interval for etcd_state_client to reload state

CMakeLists.txt

@@ -2,6 +2,6 @@ cmake_minimum_required(VERSION 2.8.12)
 
 project(vitastor)
 
-set(VITASTOR_VERSION "2.0.0")
+set(VITASTOR_VERSION "2.1.0")
 
 add_subdirectory(src)

csi/Dockerfile

@@ -37,8 +37,8 @@ RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/
     wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
     apt-get update && \
     apt-get install -y vitastor-client && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-8.1.2%2Bds-1%2Bvitastor1/qemu-utils_8.1.2%2Bds-1%2Bvitastor1_amd64.deb && \
-    wget https://vitastor.io/archive/qemu/qemu-bookworm-8.1.2%2Bds-1%2Bvitastor1/qemu-block-extra_8.1.2%2Bds-1%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-utils_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-9.2.2%2Bds-1%2Bvitastor4/qemu-block-extra_9.2.2%2Bds-1%2Bvitastor4_amd64.deb && \
     dpkg -x qemu-utils*.deb tmp1 && \
     dpkg -x qemu-block-extra*.deb tmp1 && \
     cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \

csi/Makefile

@@ -1,4 +1,4 @@
-VITASTOR_VERSION ?= v2.0.0
+VITASTOR_VERSION ?= v2.1.0
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v2.0.0
+          image: vitalif/vitastor-csi:v2.1.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -121,7 +121,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v2.0.0
+          image: vitalif/vitastor-csi:v2.1.0
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "2.0.0"
+    vitastorCSIDriverVersion = "2.1.0"
 )
 
 // Config struct fills the parameters of request or user input

debian/changelog

@@ -1,4 +1,4 @@
-vitastor (2.0.0-1) unstable; urgency=medium
+vitastor (2.1.0-1) unstable; urgency=medium
 
   * Bugfixes
 

debian/patched-qemu.Dockerfile

@@ -10,10 +10,14 @@ ARG REL=
 WORKDIR /root
 
 RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" -o "$REL" = "bookworm" ]; then \
-        echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        if [ "$REL" = "buster" ]; then \
+            echo "deb http://archive.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        else \
+            echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        fi; \
         echo >> /etc/apt/preferences; \
         echo 'Package: *' >> /etc/apt/preferences; \
-        echo "Pin: release a=$REL-backports" >> /etc/apt/preferences; \
+        echo "Pin: release n=$REL-backports" >> /etc/apt/preferences; \
         echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
     fi; \
     grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
@@ -56,7 +60,7 @@ RUN set -e; \
     quilt add block/vitastor.c; \
     cp /root/qemu_driver.c block/vitastor.c; \
     quilt refresh; \
-    V=$(head -n1 debian/changelog | perl -pe 's/5\.2\+dfsg-9/5.2+dfsg-11/; s/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor4; \
+    V=$(head -n1 debian/changelog | perl -pe 's/5\.2\+dfsg-9/5.2+dfsg-11/; s/^.*\((.*?)(\+deb\d+u\d+)?(~bpo[\d\+]*)?\).*$/$1/')+vitastor5; \
     if [ "$REL" = bullseye ]; then V=${V}bullseye; fi; \
     DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docker/Makefile

@@ -1,9 +1,9 @@
-VITASTOR_VERSION ?= v2.0.0
+VITASTOR_VERSION ?= v2.1.0
 
 all: build push
 
 build:
-	@docker build --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
+	@docker build --no-cache --rm -t vitalif/vitastor:$(VITASTOR_VERSION) .
 
 push:
 	@docker push vitalif/vitastor:$(VITASTOR_VERSION)

docker/etc/apt/sources.list.d/vitastor.list

@@ -1 +1,2 @@
 deb http://vitastor.io/debian bookworm main
+deb http://http.debian.net/debian/ bookworm-backports main

docker/etc/vitastor/docker.conf

@@ -4,7 +4,7 @@
 #
 
 # Desired Vitastor version
-VITASTOR_VERSION=v2.0.0
+VITASTOR_VERSION=v2.1.0
 
 # Additional arguments for all containers
 # For example, you may want to specify a custom logging driver here

docs/config/monitor.en.md

@@ -74,13 +74,13 @@ Grafana dashboard suitable for this exporter is here: [Vitastor-Grafana-6+.json]
 - Type: integer
 - Default: 8060
 
-HTTP port for monitors to listen on (including metrics exporter)
+HTTP port for monitors to listen to (including metrics exporter)
 
 ## mon_http_ip
 
 - Type: string
 
-IP address for monitors to listen on (all addresses by default)
+IP address for monitors to listen to (all addresses by default)
 
 ## mon_https_cert
 

docs/config/network.en.md

@@ -9,9 +9,11 @@
 These parameters apply to clients and OSDs and affect network connection logic
 between clients, OSDs and etcd.
 
-- [tcp_header_buffer_size](#tcp_header_buffer_size)
-- [use_sync_send_recv](#use_sync_send_recv)
+- [osd_network](#osd_network)
+- [osd_cluster_network](#osd_cluster_network)
 - [use_rdma](#use_rdma)
+- [use_rdmacm](#use_rdmacm)
+- [disable_tcp](#disable_tcp)
 - [rdma_device](#rdma_device)
 - [rdma_port_num](#rdma_port_num)
 - [rdma_gid_index](#rdma_gid_index)
@@ -30,38 +32,62 @@ between clients, OSDs and etcd.
 - [etcd_slow_timeout](#etcd_slow_timeout)
 - [etcd_keepalive_timeout](#etcd_keepalive_timeout)
 - [etcd_ws_keepalive_interval](#etcd_ws_keepalive_interval)
+- [etcd_min_reload_interval](#etcd_min_reload_interval)
+- [tcp_header_buffer_size](#tcp_header_buffer_size)
+- [use_sync_send_recv](#use_sync_send_recv)
 
-## tcp_header_buffer_size
+## osd_network
 
-- Type: integer
-- Default: 65536
+- Type: string or array of strings
 
-Size of the buffer used to read data using an additional copy. Vitastor
-packet headers are 128 bytes, payload is always at least 4 KB, so it is
-usually beneficial to try to read multiple packets at once even though
-it requires to copy the data an additional time. The rest of each packet
-is received without an additional copy. You can try to play with this
-parameter and see how it affects random iops and linear bandwidth if you
-want.
+Network mask of public OSD network(s) (IPv4 or IPv6). Each OSD listens to all
+addresses of UP + RUNNING interfaces matching one of these networks, on the
+same port. Port is auto-selected except if [bind_port](osd.en.md#bind_port) is
+explicitly specified. Bind address(es) may also be overridden manually by
+specifying [bind_address](osd.en.md#bind_address). If OSD networks are not specified
+at all, OSD just listens to a wildcard address (0.0.0.0).
 
-## use_sync_send_recv
+## osd_cluster_network
 
-- Type: boolean
-- Default: false
+- Type: string or array of strings
 
-If true, synchronous send/recv syscalls are used instead of io_uring for
-socket communication. Useless for OSDs because they require io_uring anyway,
-but may be required for clients with old kernel versions.
+Network mask of separate network(s) (IPv4 or IPv6) to use for OSD
+cluster connections. I.e. OSDs will always attempt to use these networks
+to connect to other OSDs, while clients will attempt to use networks from
+[osd_network](#osd_network).
 
 ## use_rdma
 
 - Type: boolean
 - Default: true
 
-Try to use RDMA for communication if it's available. Disable if you don't
-want Vitastor to use RDMA. TCP-only clients can also talk to an RDMA-enabled
-cluster, so disabling RDMA may be needed if clients have RDMA devices,
-but they are not connected to the cluster.
+Try to use RDMA through libibverbs for communication if it's available.
+Disable if you don't want Vitastor to use RDMA. TCP-only clients can also
+talk to an RDMA-enabled cluster, so disabling RDMA may be needed if clients
+have RDMA devices, but they are not connected to the cluster.
+
+`use_rdma` works with RoCEv1/RoCEv2 networks, but not with iWARP and,
+maybe, with some Infiniband configurations which require RDMA-CM.
+Consider `use_rdmacm` for such networks.
+
+## use_rdmacm
+
+- Type: boolean
+- Default: true
+
+Use an alternative implementation of RDMA through RDMA-CM (Connection
+Manager). Works with all RDMA networks: Infiniband, iWARP and
+RoCEv1/RoCEv2, and even allows to disable TCP and run only with RDMA.
+OSDs always use random port numbers for RDMA-CM listeners, different
+from their TCP ports. `use_rdma` is automatically disabled when
+`use_rdmacm` is enabled.
+
+## disable_tcp
+
+- Type: boolean
+- Default: true
+
+Fully disable TCP and only use RDMA-CM for OSD communication.
 
 ## rdma_device
 
@@ -92,12 +118,13 @@ PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
 ## rdma_port_num
 
 - Type: integer
-- Default: 1
 
 RDMA device port number to use. Only for devices that have more than 1 port.
 See `phys_port_cnt` in `ibv_devinfo -v` output to determine how many ports
 your device has.
 
+Not relevant for RDMA-CM (use_rdmacm).
+
 ## rdma_gid_index
 
 - Type: integer
@@ -113,13 +140,14 @@ GID auto-selection is unsupported with libibverbs < v32.
 
 A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
 
+Not relevant for RDMA-CM (use_rdmacm).
+
 ## rdma_mtu
 
 - Type: integer
-- Default: 4096
 
-RDMA Path MTU to use. Must be 1024, 2048 or 4096. There is usually no
-sense to change it from the default 4096.
+RDMA Path MTU to use. Must be 1024, 2048 or 4096. Default is to use the
+RDMA device's MTU.
 
 ## rdma_max_sge
 
@@ -261,3 +289,35 @@ etcd_report_interval to guarantee that keepalive actually works.
 
 etcd websocket ping interval required to keep the connection alive and
 detect disconnections quickly.
+
+## etcd_min_reload_interval
+
+- Type: milliseconds
+- Default: 1000
+- Can be changed online: yes
+
+Minimum interval for full etcd state reload. Introduced to prevent
+excessive load on etcd during outages when etcd can't keep up with event
+streams and cancels them.
+
+## tcp_header_buffer_size
+
+- Type: integer
+- Default: 65536
+
+Size of the buffer used to read data using an additional copy. Vitastor
+packet headers are 128 bytes, payload is always at least 4 KB, so it is
+usually beneficial to try to read multiple packets at once even though
+it requires to copy the data an additional time. The rest of each packet
+is received without an additional copy. You can try to play with this
+parameter and see how it affects random iops and linear bandwidth if you
+want.
+
+## use_sync_send_recv
+
+- Type: boolean
+- Default: false
+
+If true, synchronous send/recv syscalls are used instead of io_uring for
+socket communication. Useless for OSDs because they require io_uring anyway,
+but may be required for clients with old kernel versions.

docs/config/network.ru.md

@@ -9,9 +9,11 @@
 Данные параметры используются клиентами и OSD и влияют на логику сетевого
 взаимодействия между клиентами, OSD, а также etcd.
 
-- [tcp_header_buffer_size](#tcp_header_buffer_size)
-- [use_sync_send_recv](#use_sync_send_recv)
+- [osd_network](#osd_network)
+- [osd_cluster_network](#osd_cluster_network)
 - [use_rdma](#use_rdma)
+- [use_rdmacm](#use_rdmacm)
+- [disable_tcp](#disable_tcp)
 - [rdma_device](#rdma_device)
 - [rdma_port_num](#rdma_port_num)
 - [rdma_gid_index](#rdma_gid_index)
@@ -30,41 +32,62 @@
 - [etcd_slow_timeout](#etcd_slow_timeout)
 - [etcd_keepalive_timeout](#etcd_keepalive_timeout)
 - [etcd_ws_keepalive_interval](#etcd_ws_keepalive_interval)
+- [etcd_min_reload_interval](#etcd_min_reload_interval)
+- [tcp_header_buffer_size](#tcp_header_buffer_size)
+- [use_sync_send_recv](#use_sync_send_recv)
 
-## tcp_header_buffer_size
+## osd_network
 
-- Тип: целое число
-- Значение по умолчанию: 65536
+- Тип: строка или массив строк
 
-Размер буфера для чтения данных с дополнительным копированием. Пакеты
-Vitastor содержат 128-байтные заголовки, за которыми следуют данные размером
-от 4 КБ и для мелких операций ввода-вывода обычно выгодно за 1 вызов читать
-сразу несколько пакетов, даже не смотря на то, что это требует лишний раз
-скопировать данные. Часть каждого пакета за пределами значения данного
-параметра читается без дополнительного копирования. Вы можете попробовать
-поменять этот параметр и посмотреть, как он влияет на производительность
-случайного и линейного доступа.
+Маски подсетей (IPv4 или IPv6) публичной сети или сетей OSD. Каждый OSD слушает
+один и тот же порт на всех адресах поднятых (UP + RUNNING) сетевых интерфейсов,
+соответствующих одной из указанных сетей. Порт выбирается автоматически, если
+только [bind_port](osd.ru.md#bind_port) не задан явно. Адреса для подключений можно
+также переопределить явно, задав [bind_address](osd.ru.md#bind_address). Если сети OSD
+не заданы вообще, OSD слушает все адреса (0.0.0.0).
 
-## use_sync_send_recv
+## osd_cluster_network
 
-- Тип: булево (да/нет)
-- Значение по умолчанию: false
+- Тип: строка или массив строк
 
-Если установлено в истину, то вместо io_uring для передачи данных по сети
-будут использоваться обычные синхронные системные вызовы send/recv. Для OSD
-это бессмысленно, так как OSD в любом случае нуждается в io_uring, но, в
-принципе, это может применяться для клиентов со старыми версиями ядра.
+Маски подсетей (IPv4 или IPv6) отдельной кластерной сети или сетей OSD.
+То есть, OSD будут всегда стараться использовать эти сети для соединений
+с другими OSD, а клиенты будут стараться использовать сети из [osd_network](#osd_network).
 
 ## use_rdma
 
 - Тип: булево (да/нет)
 - Значение по умолчанию: true
 
-Пытаться использовать RDMA для связи при наличии доступных устройств.
-Отключите, если вы не хотите, чтобы Vitastor использовал RDMA.
-TCP-клиенты также могут работать с RDMA-кластером, так что отключать
-RDMA может быть нужно только если у клиентов есть RDMA-устройства,
-но они не имеют соединения с кластером Vitastor.
+Попробовать использовать RDMA через libibverbs для связи при наличии
+доступных устройств. Отключите, если вы не хотите, чтобы Vitastor
+использовал RDMA. TCP-клиенты также могут работать с RDMA-кластером,
+так что отключать RDMA может быть нужно, только если у клиентов есть
+RDMA-устройства, но они не имеют соединения с кластером Vitastor.
+
+`use_rdma` работает с RoCEv1/RoCEv2 сетями, но не работает с iWARP и
+может не работать с частью конфигураций Infiniband, требующих RDMA-CM.
+Рассмотрите включение `use_rdmacm` для таких сетей.
+
+## use_rdmacm
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: true
+
+Использовать альтернативную реализацию RDMA на основе RDMA-CM (Connection
+Manager). Работает со всеми типами RDMA-сетей: Infiniband, iWARP и
+RoCEv1/RoCEv2, и даже позволяет полностью отключить TCP и работать
+только на RDMA. OSD используют случайные номера портов для ожидания
+соединений через RDMA-CM, отличающиеся от их TCP-портов. Также при
+включении `use_rdmacm` автоматически отключается опция `use_rdma`.
+
+## disable_tcp
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: true
+
+Полностью отключить TCP и использовать только RDMA-CM для соединений с OSD.
 
 ## rdma_device
 
@@ -96,13 +119,14 @@ Control) и ECN (Explicit Congestion Notification).
 ## rdma_port_num
 
 - Тип: целое число
-- Значение по умолчанию: 1
 
 Номер порта RDMA-устройства, который следует использовать. Имеет смысл
 только для устройств, у которых более 1 порта. Чтобы узнать, сколько портов
 у вашего адаптера, посмотрите `phys_port_cnt` в выводе команды
 `ibv_devinfo -v`.
 
+Опция неприменима к RDMA-CM (use_rdmacm).
+
 ## rdma_gid_index
 
 - Тип: целое число
@@ -119,13 +143,14 @@ libibverbs < v32.
 
 Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
 
+Опция неприменима к RDMA-CM (use_rdmacm).
+
 ## rdma_mtu
 
 - Тип: целое число
-- Значение по умолчанию: 4096
 
 Максимальная единица передачи (Path MTU) для RDMA. Должно быть равно 1024,
-2048 или 4096. Обычно нет смысла менять значение по умолчанию, равное 4096.
+2048 или 4096. По умолчанию используется значение MTU RDMA-устройства.
 
 ## rdma_max_sge
 
@@ -271,3 +296,37 @@ etcd_report_interval, чтобы keepalive гарантированно рабо
 - Можно менять на лету: да
 
 Интервал проверки живости вебсокет-подключений к etcd.
+
+## etcd_min_reload_interval
+
+- Тип: миллисекунды
+- Значение по умолчанию: 1000
+- Можно менять на лету: да
+
+Минимальный интервал полной перезагрузки состояния из etcd. Добавлено для
+предотвращения избыточной нагрузки на etcd во время отказов, когда etcd не
+успевает рассылать потоки событий и отменяет их.
+
+## tcp_header_buffer_size
+
+- Тип: целое число
+- Значение по умолчанию: 65536
+
+Размер буфера для чтения данных с дополнительным копированием. Пакеты
+Vitastor содержат 128-байтные заголовки, за которыми следуют данные размером
+от 4 КБ и для мелких операций ввода-вывода обычно выгодно за 1 вызов читать
+сразу несколько пакетов, даже не смотря на то, что это требует лишний раз
+скопировать данные. Часть каждого пакета за пределами значения данного
+параметра читается без дополнительного копирования. Вы можете попробовать
+поменять этот параметр и посмотреть, как он влияет на производительность
+случайного и линейного доступа.
+
+## use_sync_send_recv
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Если установлено в истину, то вместо io_uring для передачи данных по сети
+будут использоваться обычные синхронные системные вызовы send/recv. Для OSD
+это бессмысленно, так как OSD в любом случае нуждается в io_uring, но, в
+принципе, это может применяться для клиентов со старыми версиями ядра.

docs/config/osd.en.md

@@ -10,13 +10,12 @@ These parameters only apply to OSDs, are not fixed at the moment of OSD drive
 initialization and can be changed - in /etc/vitastor/vitastor.conf or [vitastor-disk update-sb](../usage/disk.en.md#update-sb)
 with an OSD restart or, for some of them, even without restarting by updating configuration in etcd.
 
+- [bind_address](#bind_address)
+- [bind_port](#bind_port)
 - [osd_iothread_count](#osd_iothread_count)
 - [etcd_report_interval](#etcd_report_interval)
 - [etcd_stats_interval](#etcd_stats_interval)
 - [run_primary](#run_primary)
-- [osd_network](#osd_network)
-- [bind_address](#bind_address)
-- [bind_port](#bind_port)
 - [autosync_interval](#autosync_interval)
 - [autosync_writes](#autosync_writes)
 - [recovery_queue_depth](#recovery_queue_depth)
@@ -63,6 +62,24 @@ with an OSD restart or, for some of them, even without restarting by updating co
 - [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
 - [discard_on_start](#discard_on_start)
 - [min_discard_size](#min_discard_size)
+- [allow_net_split](#allow_net_split)
+
+## bind_address
+
+- Type: string or array of strings
+
+Instead of the network masks ([osd_network](network.en.md#osd_network) and
+[osd_cluster_network](network.en.md#osd_cluster_network)), you can also set
+OSD listen addresses explicitly using this parameter. May be useful if you
+want to start OSDs on interfaces that are not UP + RUNNING.
+
+## bind_port
+
+- Type: integer
+
+By default, OSDs pick random ports to use for incoming connections
+automatically. With this option you can set a specific port for a specific
+OSD by hand.
 
 ## osd_iothread_count
 
@@ -106,34 +123,6 @@ debugging purposes. It's possible to implement additional feature for the
 monitor which may allow to separate primary and secondary OSDs, but it's
 unclear why anyone could need it, so it's not implemented.
 
-## osd_network
-
-- Type: string or array of strings
-
-Network mask of the network (IPv4 or IPv6) to use for OSDs. Note that
-although it's possible to specify multiple networks here, this does not
-mean that OSDs will create multiple listening sockets - they'll only
-pick the first matching address of an UP + RUNNING interface. Separate
-networks for cluster and client connections are also not implemented, but
-they are mostly useless anyway, so it's not a big deal.
-
-## bind_address
-
-- Type: string
-- Default: 0.0.0.0
-
-Instead of the network mask, you can also set OSD listen address explicitly
-using this parameter. May be useful if you want to start OSDs on interfaces
-that are not UP + RUNNING.
-
-## bind_port
-
-- Type: integer
-
-By default, OSDs pick random ports to use for incoming connections
-automatically. With this option you can set a specific port for a specific
-OSD by hand.
-
 ## autosync_interval
 
 - Type: seconds
@@ -644,3 +633,17 @@ Discard (SSD TRIM) unused data device blocks on every OSD startup.
 - Default: 1048576
 
 Minimum consecutive block size to TRIM it.
+
+## allow_net_split
+
+- Type: boolean
+- Default: false
+
+Allow "safe" cases of network splits/partitions - allow to start PGs without
+connections to some OSDs currently registered as alive in etcd, if the number
+of actually connected PG OSDs is at least pg_minsize. That is, allow some OSDs to lose
+connectivity with some other OSDs as long as it doesn't break pg_minsize guarantees.
+The downside is that it increases the probability of writing data into just pg_minsize
+OSDs during failover which can lead to PGs becoming incomplete after additional outages.
+
+The old behaviour in versions up to 2.0.0 was equal to enabled allow_net_split.

docs/config/osd.ru.md

@@ -11,13 +11,12 @@
 момент с перезапуском OSD в /etc/vitastor/vitastor.conf или [vitastor-disk update-sb](../usage/disk.ru.md#update-sb),
 а некоторые и без перезапуска, с помощью изменения конфигурации в etcd.
 
+- [bind_address](#bind_address)
+- [bind_port](#bind_port)
 - [osd_iothread_count](#osd_iothread_count)
 - [etcd_report_interval](#etcd_report_interval)
 - [etcd_stats_interval](#etcd_stats_interval)
 - [run_primary](#run_primary)
-- [osd_network](#osd_network)
-- [bind_address](#bind_address)
-- [bind_port](#bind_port)
 - [autosync_interval](#autosync_interval)
 - [autosync_writes](#autosync_writes)
 - [recovery_queue_depth](#recovery_queue_depth)
@@ -64,6 +63,24 @@
 - [recovery_tune_sleep_cutoff_us](#recovery_tune_sleep_cutoff_us)
 - [discard_on_start](#discard_on_start)
 - [min_discard_size](#min_discard_size)
+- [allow_net_split](#allow_net_split)
+
+## bind_address
+
+- Тип: строка или массив строк
+
+Вместо использования масок подсети ([osd_network](network.ru.md#osd_network) и
+[osd_cluster_network](network.ru.md#osd_cluster_network)), вы также можете явно
+задать адрес(а), на которых будут ожидать соединений OSD, с помощью данного
+параметра. Это может быть полезно, например, чтобы запускать OSD на неподнятых
+интерфейсах (не UP + RUNNING).
+
+## bind_port
+
+- Тип: целое число
+
+По умолчанию OSD сами выбирают случайные порты для входящих подключений.
+С помощью данной опции вы можете задать порт для отдельного OSD вручную.
 
 ## osd_iothread_count
 
@@ -109,34 +126,6 @@ max_etcd_attempts * etcd_quick_timeout.
 первичные OSD от вторичных, но пока не понятно, зачем это может кому-то
 понадобиться, поэтому это не реализовано.
 
-## osd_network
-
-- Тип: строка или массив строк
-
-Маска подсети (IPv4 или IPv6) для использования для соединений с OSD.
-Имейте в виду, что хотя сейчас и можно передать в этот параметр несколько
-подсетей, это не означает, что OSD будут создавать несколько слушающих
-сокетов - они лишь будут выбирать адрес первого поднятого (состояние UP +
-RUNNING), подходящий под заданную маску. Также не реализовано разделение
-кластерной и публичной сетей OSD. Правда, от него обычно всё равно довольно
-мало толку, так что особенной проблемы в этом нет.
-
-## bind_address
-
-- Тип: строка
-- Значение по умолчанию: 0.0.0.0
-
-Этим параметром можно явным образом задать адрес, на котором будет ожидать
-соединений OSD (вместо использования маски подсети). Может быть полезно,
-например, чтобы запускать OSD на неподнятых интерфейсах (не UP + RUNNING).
-
-## bind_port
-
-- Тип: целое число
-
-По умолчанию OSD сами выбирают случайные порты для входящих подключений.
-С помощью данной опции вы можете задать порт для отдельного OSD вручную.
-
 ## autosync_interval
 
 - Тип: секунды
@@ -675,3 +664,18 @@ EC (кодов коррекции ошибок) с более, чем 1 диск
 - Значение по умолчанию: 1048576
 
 Минимальный размер последовательного блока данных, чтобы освобождать его через TRIM.
+
+## allow_net_split
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+
+Разрешить "безопасные" случаи разделений сети - разрешить активировать PG без
+соединений к некоторым OSD, помеченным активными в etcd, если общее число активных
+OSD в PG составляет как минимум pg_minsize. То есть, разрешать некоторым OSD терять
+соединения с некоторыми другими OSD, если это не нарушает гарантий pg_minsize.
+Минус такого разрешения в том, что оно повышает вероятность записи данных ровно в
+pg_minsize OSD во время переключений, что может потом привести к тому, что PG станут
+неполными (incomplete), если упадут ещё какие-то OSD.
+
+Старое поведение в версиях до 2.0.0 было идентично включённому allow_net_split.

docs/config/pool.en.md

@@ -189,6 +189,9 @@ So, pg_minsize regulates the number of failures that a pool can tolerate
 without temporary downtime for [osd_out_time](monitor.en.md#osd_out_time),
 but at a cost of slightly reduced storage reliability.
 
+See also [allow_net_split](osd.en.md#allow_net_split) and
+[PG state descriptions](../usage/admin.en.md#pg-states).
+
 FIXME: pg_minsize behaviour may be changed in the future to only make PGs
 read-only instead of deactivating them.
 

docs/config/pool.ru.md

@@ -413,6 +413,9 @@ OSD с "all".
 (например, `s3:standard`) - конкретное содержимое `<name>` пока никак не проверяется
 компонентами Vitastor S3.
 
+Смотрите также [allow_net_split](osd.ru.md#allow_net_split) и
+[документацию по состояниям PG](../usage/admin.ru.md#состояния-pg).
+
 Все остальные значения used_for_app, кроме начинающихся на `fs:` или `s3:`, не
 означают ничего особенного для основных компонентов Vitastor. Поэтому сейчас вы
 можете использовать их свободно любым желаемым способом.

docs/config/src/monitor.yml

@@ -75,11 +75,11 @@
 - name: mon_http_port
   type: int
   default: 8060
-  info: HTTP port for monitors to listen on (including metrics exporter)
+  info: HTTP port for monitors to listen to (including metrics exporter)
   info_ru: Порт, на котором мониторы принимают HTTP-соединения (в том числе для отдачи метрик)
 - name: mon_http_ip
   type: string
-  info: IP address for monitors to listen on (all addresses by default)
+  info: IP address for monitors to listen to (all addresses by default)
   info_ru: IP-адрес, на котором мониторы принимают HTTP-соединения (по умолчанию все адреса)
 - name: mon_https_cert
   type: string

docs/config/src/network.yml

@@ -1,49 +1,78 @@
-- name: tcp_header_buffer_size
-  type: int
-  default: 65536
+- name: osd_network
+  type: string or array of strings
+  type_ru: строка или массив строк
   info: |
-    Size of the buffer used to read data using an additional copy. Vitastor
-    packet headers are 128 bytes, payload is always at least 4 KB, so it is
-    usually beneficial to try to read multiple packets at once even though
-    it requires to copy the data an additional time. The rest of each packet
-    is received without an additional copy. You can try to play with this
-    parameter and see how it affects random iops and linear bandwidth if you
-    want.
+    Network mask of public OSD network(s) (IPv4 or IPv6). Each OSD listens to all
+    addresses of UP + RUNNING interfaces matching one of these networks, on the
+    same port. Port is auto-selected except if [bind_port](osd.en.md#bind_port) is
+    explicitly specified. Bind address(es) may also be overridden manually by
+    specifying [bind_address](osd.en.md#bind_address). If OSD networks are not specified
+    at all, OSD just listens to a wildcard address (0.0.0.0).
   info_ru: |
-    Размер буфера для чтения данных с дополнительным копированием. Пакеты
-    Vitastor содержат 128-байтные заголовки, за которыми следуют данные размером
-    от 4 КБ и для мелких операций ввода-вывода обычно выгодно за 1 вызов читать
-    сразу несколько пакетов, даже не смотря на то, что это требует лишний раз
-    скопировать данные. Часть каждого пакета за пределами значения данного
-    параметра читается без дополнительного копирования. Вы можете попробовать
-    поменять этот параметр и посмотреть, как он влияет на производительность
-    случайного и линейного доступа.
-- name: use_sync_send_recv
-  type: bool
-  default: false
+    Маски подсетей (IPv4 или IPv6) публичной сети или сетей OSD. Каждый OSD слушает
+    один и тот же порт на всех адресах поднятых (UP + RUNNING) сетевых интерфейсов,
+    соответствующих одной из указанных сетей. Порт выбирается автоматически, если
+    только [bind_port](osd.ru.md#bind_port) не задан явно. Адреса для подключений можно
+    также переопределить явно, задав [bind_address](osd.ru.md#bind_address). Если сети OSD
+    не заданы вообще, OSD слушает все адреса (0.0.0.0).
+- name: osd_cluster_network
+  type: string or array of strings
+  type_ru: строка или массив строк
   info: |
-    If true, synchronous send/recv syscalls are used instead of io_uring for
-    socket communication. Useless for OSDs because they require io_uring anyway,
-    but may be required for clients with old kernel versions.
+    Network mask of separate network(s) (IPv4 or IPv6) to use for OSD
+    cluster connections. I.e. OSDs will always attempt to use these networks
+    to connect to other OSDs, while clients will attempt to use networks from
+    [osd_network](#osd_network).
   info_ru: |
-    Если установлено в истину, то вместо io_uring для передачи данных по сети
-    будут использоваться обычные синхронные системные вызовы send/recv. Для OSD
-    это бессмысленно, так как OSD в любом случае нуждается в io_uring, но, в
-    принципе, это может применяться для клиентов со старыми версиями ядра.
+    Маски подсетей (IPv4 или IPv6) отдельной кластерной сети или сетей OSD.
+    То есть, OSD будут всегда стараться использовать эти сети для соединений
+    с другими OSD, а клиенты будут стараться использовать сети из [osd_network](#osd_network).
 - name: use_rdma
   type: bool
   default: true
   info: |
-    Try to use RDMA for communication if it's available. Disable if you don't
-    want Vitastor to use RDMA. TCP-only clients can also talk to an RDMA-enabled
-    cluster, so disabling RDMA may be needed if clients have RDMA devices,
-    but they are not connected to the cluster.
+    Try to use RDMA through libibverbs for communication if it's available.
+    Disable if you don't want Vitastor to use RDMA. TCP-only clients can also
+    talk to an RDMA-enabled cluster, so disabling RDMA may be needed if clients
+    have RDMA devices, but they are not connected to the cluster.
+
+    `use_rdma` works with RoCEv1/RoCEv2 networks, but not with iWARP and,
+    maybe, with some Infiniband configurations which require RDMA-CM.
+    Consider `use_rdmacm` for such networks.
   info_ru: |
-    Пытаться использовать RDMA для связи при наличии доступных устройств.
-    Отключите, если вы не хотите, чтобы Vitastor использовал RDMA.
-    TCP-клиенты также могут работать с RDMA-кластером, так что отключать
-    RDMA может быть нужно только если у клиентов есть RDMA-устройства,
-    но они не имеют соединения с кластером Vitastor.
+    Попробовать использовать RDMA через libibverbs для связи при наличии
+    доступных устройств. Отключите, если вы не хотите, чтобы Vitastor
+    использовал RDMA. TCP-клиенты также могут работать с RDMA-кластером,
+    так что отключать RDMA может быть нужно, только если у клиентов есть
+    RDMA-устройства, но они не имеют соединения с кластером Vitastor.
+
+    `use_rdma` работает с RoCEv1/RoCEv2 сетями, но не работает с iWARP и
+    может не работать с частью конфигураций Infiniband, требующих RDMA-CM.
+    Рассмотрите включение `use_rdmacm` для таких сетей.
+- name: use_rdmacm
+  type: bool
+  default: true
+  info: |
+    Use an alternative implementation of RDMA through RDMA-CM (Connection
+    Manager). Works with all RDMA networks: Infiniband, iWARP and
+    RoCEv1/RoCEv2, and even allows to disable TCP and run only with RDMA.
+    OSDs always use random port numbers for RDMA-CM listeners, different
+    from their TCP ports. `use_rdma` is automatically disabled when
+    `use_rdmacm` is enabled.
+  info_ru: |
+    Использовать альтернативную реализацию RDMA на основе RDMA-CM (Connection
+    Manager). Работает со всеми типами RDMA-сетей: Infiniband, iWARP и
+    RoCEv1/RoCEv2, и даже позволяет полностью отключить TCP и работать
+    только на RDMA. OSD используют случайные номера портов для ожидания
+    соединений через RDMA-CM, отличающиеся от их TCP-портов. Также при
+    включении `use_rdmacm` автоматически отключается опция `use_rdma`.
+- name: disable_tcp
+  type: bool
+  default: true
+  info: |
+    Fully disable TCP and only use RDMA-CM for OSD communication.
+  info_ru: |
+    Полностью отключить TCP и использовать только RDMA-CM для соединений с OSD.
 - name: rdma_device
   type: string
   info: |
@@ -93,16 +122,19 @@
     Control) и ECN (Explicit Congestion Notification).
 - name: rdma_port_num
   type: int
-  default: 1
   info: |
     RDMA device port number to use. Only for devices that have more than 1 port.
     See `phys_port_cnt` in `ibv_devinfo -v` output to determine how many ports
     your device has.
+
+    Not relevant for RDMA-CM (use_rdmacm).
   info_ru: |
     Номер порта RDMA-устройства, который следует использовать. Имеет смысл
     только для устройств, у которых более 1 порта. Чтобы узнать, сколько портов
     у вашего адаптера, посмотрите `phys_port_cnt` в выводе команды
     `ibv_devinfo -v`.
+
+    Опция неприменима к RDMA-CM (use_rdmacm).
 - name: rdma_gid_index
   type: int
   info: |
@@ -116,6 +148,8 @@
     GID auto-selection is unsupported with libibverbs < v32.
 
     A correct rdma_gid_index for RoCEv2 is usually 1 (IPv6) or 3 (IPv4).
+
+    Not relevant for RDMA-CM (use_rdmacm).
   info_ru: |
     Номер глобального идентификатора адреса RDMA-устройства, который следует
     использовать. Разным gid_index могут соответствовать разные протоколы связи:
@@ -128,15 +162,16 @@
     libibverbs < v32.
 
     Правильный rdma_gid_index для RoCEv2, как правило, 1 (IPv6) или 3 (IPv4).
+
+    Опция неприменима к RDMA-CM (use_rdmacm).
 - name: rdma_mtu
   type: int
-  default: 4096
   info: |
-    RDMA Path MTU to use. Must be 1024, 2048 or 4096. There is usually no
-    sense to change it from the default 4096.
+    RDMA Path MTU to use. Must be 1024, 2048 or 4096. Default is to use the
+    RDMA device's MTU.
   info_ru: |
     Максимальная единица передачи (Path MTU) для RDMA. Должно быть равно 1024,
-    2048 или 4096. Обычно нет смысла менять значение по умолчанию, равное 4096.
+    2048 или 4096. По умолчанию используется значение MTU RDMA-устройства.
 - name: rdma_max_sge
   type: int
   default: 128
@@ -306,3 +341,47 @@
     detect disconnections quickly.
   info_ru: |
     Интервал проверки живости вебсокет-подключений к etcd.
+- name: etcd_min_reload_interval
+  type: ms
+  default: 1000
+  online: true
+  info: |
+    Minimum interval for full etcd state reload. Introduced to prevent
+    excessive load on etcd during outages when etcd can't keep up with event
+    streams and cancels them.
+  info_ru: |
+    Минимальный интервал полной перезагрузки состояния из etcd. Добавлено для
+    предотвращения избыточной нагрузки на etcd во время отказов, когда etcd не
+    успевает рассылать потоки событий и отменяет их.
+- name: tcp_header_buffer_size
+  type: int
+  default: 65536
+  info: |
+    Size of the buffer used to read data using an additional copy. Vitastor
+    packet headers are 128 bytes, payload is always at least 4 KB, so it is
+    usually beneficial to try to read multiple packets at once even though
+    it requires to copy the data an additional time. The rest of each packet
+    is received without an additional copy. You can try to play with this
+    parameter and see how it affects random iops and linear bandwidth if you
+    want.
+  info_ru: |
+    Размер буфера для чтения данных с дополнительным копированием. Пакеты
+    Vitastor содержат 128-байтные заголовки, за которыми следуют данные размером
+    от 4 КБ и для мелких операций ввода-вывода обычно выгодно за 1 вызов читать
+    сразу несколько пакетов, даже не смотря на то, что это требует лишний раз
+    скопировать данные. Часть каждого пакета за пределами значения данного
+    параметра читается без дополнительного копирования. Вы можете попробовать
+    поменять этот параметр и посмотреть, как он влияет на производительность
+    случайного и линейного доступа.
+- name: use_sync_send_recv
+  type: bool
+  default: false
+  info: |
+    If true, synchronous send/recv syscalls are used instead of io_uring for
+    socket communication. Useless for OSDs because they require io_uring anyway,
+    but may be required for clients with old kernel versions.
+  info_ru: |
+    Если установлено в истину, то вместо io_uring для передачи данных по сети
+    будут использоваться обычные синхронные системные вызовы send/recv. Для OSD
+    это бессмысленно, так как OSD в любом случае нуждается в io_uring, но, в
+    принципе, это может применяться для клиентов со старыми версиями ядра.

docs/config/src/osd.yml

@@ -1,3 +1,26 @@
+- name: bind_address
+  type: string or array of strings
+  type_ru: строка или массив строк
+  info: |
+    Instead of the network masks ([osd_network](network.en.md#osd_network) and
+    [osd_cluster_network](network.en.md#osd_cluster_network)), you can also set
+    OSD listen addresses explicitly using this parameter. May be useful if you
+    want to start OSDs on interfaces that are not UP + RUNNING.
+  info_ru: |
+    Вместо использования масок подсети ([osd_network](network.ru.md#osd_network) и
+    [osd_cluster_network](network.ru.md#osd_cluster_network)), вы также можете явно
+    задать адрес(а), на которых будут ожидать соединений OSD, с помощью данного
+    параметра. Это может быть полезно, например, чтобы запускать OSD на неподнятых
+    интерфейсах (не UP + RUNNING).
+- name: bind_port
+  type: int
+  info: |
+    By default, OSDs pick random ports to use for incoming connections
+    automatically. With this option you can set a specific port for a specific
+    OSD by hand.
+  info_ru: |
+    По умолчанию OSD сами выбирают случайные порты для входящих подключений.
+    С помощью данной опции вы можете задать порт для отдельного OSD вручную.
 - name: osd_iothread_count
   type: int
   default: 0
@@ -56,44 +79,6 @@
     реализовать дополнительный режим для монитора, который позволит отделять
     первичные OSD от вторичных, но пока не понятно, зачем это может кому-то
     понадобиться, поэтому это не реализовано.
-- name: osd_network
-  type: string or array of strings
-  type_ru: строка или массив строк
-  info: |
-    Network mask of the network (IPv4 or IPv6) to use for OSDs. Note that
-    although it's possible to specify multiple networks here, this does not
-    mean that OSDs will create multiple listening sockets - they'll only
-    pick the first matching address of an UP + RUNNING interface. Separate
-    networks for cluster and client connections are also not implemented, but
-    they are mostly useless anyway, so it's not a big deal.
-  info_ru: |
-    Маска подсети (IPv4 или IPv6) для использования для соединений с OSD.
-    Имейте в виду, что хотя сейчас и можно передать в этот параметр несколько
-    подсетей, это не означает, что OSD будут создавать несколько слушающих
-    сокетов - они лишь будут выбирать адрес первого поднятого (состояние UP +
-    RUNNING), подходящий под заданную маску. Также не реализовано разделение
-    кластерной и публичной сетей OSD. Правда, от него обычно всё равно довольно
-    мало толку, так что особенной проблемы в этом нет.
-- name: bind_address
-  type: string
-  default: "0.0.0.0"
-  info: |
-    Instead of the network mask, you can also set OSD listen address explicitly
-    using this parameter. May be useful if you want to start OSDs on interfaces
-    that are not UP + RUNNING.
-  info_ru: |
-    Этим параметром можно явным образом задать адрес, на котором будет ожидать
-    соединений OSD (вместо использования маски подсети). Может быть полезно,
-    например, чтобы запускать OSD на неподнятых интерфейсах (не UP + RUNNING).
-- name: bind_port
-  type: int
-  info: |
-    By default, OSDs pick random ports to use for incoming connections
-    automatically. With this option you can set a specific port for a specific
-    OSD by hand.
-  info_ru: |
-    По умолчанию OSD сами выбирают случайные порты для входящих подключений.
-    С помощью данной опции вы можете задать порт для отдельного OSD вручную.
 - name: autosync_interval
   type: sec
   default: 5
@@ -774,3 +759,25 @@
   default: 1048576
   info: Minimum consecutive block size to TRIM it.
   info_ru: Минимальный размер последовательного блока данных, чтобы освобождать его через TRIM.
+- name: allow_net_split
+  type: bool
+  default: false
+  info: |
+    Allow "safe" cases of network splits/partitions - allow to start PGs without
+    connections to some OSDs currently registered as alive in etcd, if the number
+    of actually connected PG OSDs is at least pg_minsize. That is, allow some OSDs to lose
+    connectivity with some other OSDs as long as it doesn't break pg_minsize guarantees.
+    The downside is that it increases the probability of writing data into just pg_minsize
+    OSDs during failover which can lead to PGs becoming incomplete after additional outages.
+
+    The old behaviour in versions up to 2.0.0 was equal to enabled allow_net_split.
+  info_ru: |
+    Разрешить "безопасные" случаи разделений сети - разрешить активировать PG без
+    соединений к некоторым OSD, помеченным активными в etcd, если общее число активных
+    OSD в PG составляет как минимум pg_minsize. То есть, разрешать некоторым OSD терять
+    соединения с некоторыми другими OSD, если это не нарушает гарантий pg_minsize.
+    Минус такого разрешения в том, что оно повышает вероятность записи данных ровно в
+    pg_minsize OSD во время переключений, что может потом привести к тому, что PG станут
+    неполными (incomplete), если упадут ещё какие-то OSD.
+
+    Старое поведение в версиях до 2.0.0 было идентично включённому allow_net_split.

docs/installation/docker.en.md

@@ -26,9 +26,9 @@ at Vitastor Kubernetes operator: https://github.com/Antilles7227/vitastor-operat
 The instruction is very simple.
 
 1. Download a Docker image of the desired version: \
-   `docker pull vitastor:1.10.2`
+   `docker pull vitastor:2.1.0`
 2. Install scripts to the host system: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:2.1.0 install.sh`
 3. Reload udev rules: \
    `udevadm control --reload-rules`
 

docs/installation/docker.ru.md

@@ -25,9 +25,9 @@ Vitastor можно установить в Docker/Podman. При этом etcd,
 Инструкция по установке максимально простая.
 
 1. Скачайте Docker-образ желаемой версии: \
-   `docker pull vitastor:1.10.2`
+   `docker pull vitastor:2.1.0`
 2. Установите скрипты в хост-систему командой: \
-   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:1.10.2 install.sh`
+   `docker run --rm -it -v /etc:/host-etc -v /usr/bin:/host-bin vitastor:2.1.0 install.sh`
 3. Перезагрузите правила udev: \
    `udevadm control --reload-rules`
 

docs/installation/source.en.md

@@ -16,7 +16,7 @@
   designated initializers support from C++20
 - CMake
 - liburing, jerasure headers and libraries
-- ISA-L, libibverbs headers and libraries (optional)
+- ISA-L, libibverbs and librdmacm headers and libraries (optional)
 - tcmalloc (google-perftools-dev)
 
 ## Basic instructions

docs/installation/source.ru.md

@@ -16,7 +16,7 @@
   назначенных инициализаторов (designated initializers) из C++20
 - CMake
 - Заголовки и библиотеки liburing, jerasure
-- Опционально - заголовки и библиотеки ISA-L, libibverbs
+- Опционально - заголовки и библиотеки ISA-L, libibverbs, librdmacm
 - tcmalloc (google-perftools-dev)
 
 ## Базовая инструкция

docs/intro/features.en.md

@@ -28,7 +28,7 @@
 - Per-OSD and per-image I/O and space usage statistics in etcd
 - Snapshots and copy-on-write image clones
 - [Write throttling to smooth random write workloads in SSD+HDD configurations](../config/osd.en.md#throttle_small_writes)
-- [RDMA/RoCEv2 support via libibverbs](../config/network.en.md#rdma_device)
+- RDMA/RoCEv2 support [via libibverbs](../config/network.en.md#use_rdma) or [RDMA-CM](../config/network.en.md#use_rdmacm)
 - [Scrubbing](../config/osd.en.md#auto_scrub) (verification of copies)
 - [Checksums](../config/layout-osd.en.md#data_csum_type)
 - [Client write-back cache](../config/client.en.md#client_enable_writeback)

docs/intro/features.ru.md

@@ -30,7 +30,7 @@
 - Именование инодов через хранение их метаданных в etcd
 - Снапшоты и copy-on-write клоны
 - [Сглаживание производительности случайной записи в SSD+HDD конфигурациях](../config/osd.ru.md#throttle_small_writes)
-- [Поддержка RDMA/RoCEv2 через libibverbs](../config/network.ru.md#rdma_device)
+- Поддержка RDMA/RoCEv2 [через libibverbs](../config/network.ru.md#use_rdma) или [RDMA-CM](../config/network.ru.md#use_rdmacm)
 - [Фоновая проверка целостности](../config/osd.ru.md#auto_scrub) (сверка копий)
 - [Контрольные суммы](../config/layout-osd.ru.md#data_csum_type)
 - [Буферизация записи на стороне клиента](../config/client.ru.md#client_enable_writeback)

docs/intro/quickstart.en.md

@@ -50,7 +50,7 @@ On the monitor hosts:
 
 ## Configure OSDs
 
-- Put etcd_address and osd_network into `/etc/vitastor/vitastor.conf`. Example:
+- Put etcd_address and [osd_network](../config/network.en.md#osd_network) into `/etc/vitastor/vitastor.conf`. Example:
   ```
   {
     "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"],

docs/intro/quickstart.ru.md

@@ -50,7 +50,7 @@
 
 ## Настройте OSD
 
-- Пропишите etcd_address и osd_network в `/etc/vitastor/vitastor.conf`. Например:
+- Пропишите etcd_address и [osd_network](../config/network.ru.md#osd_network) в `/etc/vitastor/vitastor.conf`. Например:
   ```
   {
     "etcd_address": ["10.200.1.10:2379","10.200.1.11:2379","10.200.1.12:2379"],

docs/usage/admin.en.md

@@ -35,10 +35,19 @@ PG state consists of exactly 1 base state and an arbitrary number of additional
 
 PG state always includes exactly 1 of the following base states:
 - **active** — PG is active and handles user I/O.
-- **incomplete** — Not enough OSDs are available to activate this PG. That is, more disks
-  are lost than it's allowed by the pool's redundancy scheme. For example, if the pool has
-  pg_size=3 and pg_minsize=1, part of the data may be written only to 1 OSD. If that exact
-  OSD is lost, PG will become **incomplete**.
+- **incomplete** — Not enough OSDs are available to activate this PG. More exactly, that
+  means one of the following:
+  - Less than pg_minsize current target OSDs are available for the PG. I.e. more disks
+    are lost than allowed by the pool's redundancy scheme.
+  - All OSDs of some of PG's history records are unavailable, or, for EC pools, less
+    than (pg_size-parity_chunks) OSDs are available in one of the history records.
+    In other words it means that some data in this PG was written to an OSD set such that
+    it's currently impossible to read it back because these OSDs are down. For example,
+    if the pool has pg_size=3 and pg_minsize=1, part of the data may be written only to
+    1 OSD. If that exact OSD is lost, PG becomes **incomplete**.
+  - [allow_net_split](../config/osd.en.md#allow_net_split) is disabled (default) and
+    primary OSD of the PG can't connect to some secondary OSDs marked as alive in etcd.
+    I.e. a network partition happened: OSDs can talk to etcd, but not to some other OSDs.
 - **offline** — PG isn't activated by any OSD at all. Either primary OSD isn't set for
   this PG at all (if the pool is just created), or an unavailable OSD is set as primary,
   or the primary OSD refuses to start this PG (for example, because of wrong block_size),

docs/usage/admin.ru.md

@@ -35,10 +35,20 @@
 
 Состояние PG включает в себя ровно 1 флаг из следующих:
 - **active** — PG активна и обрабатывает запросы ввода-вывода от пользователей.
-- **incomplete** — Недостаточно живых OSD, чтобы включить эту PG.
-  То есть, дисков потеряно больше, чем разрешено схемой отказоустойчивости пула и pg_minsize.
-  Например, если у пула pg_size=3 и pg_minsize=1, то часть данных может записаться всего на 1 OSD.
-  Если потом конкретно этот OSD упадёт, PG окажется **incomplete**.
+- **incomplete** — Недостаточно живых OSD, чтобы включить эту PG. Если точнее, то это
+  означает один из следующих вариантов:
+  - Доступно менее, чем pg_minsize текущих целевых OSD данной PG. Иными словами, потеряно
+    больше дисков, чем это разрешает схема отказоустойчивости пула.
+  - Все OSD одной из исторических записей PG недоступны, или, для EC-пулов, в одной
+    из исторических записей PG доступно менее, чем (pg_size-parity_chunks) OSD. Другими
+    словами это означает, что часть данных этой PG была записана в такой набор OSD, из
+    которого их сейчас невозможно прочитать обратно, так как OSD не включены. Например,
+    если у пула pg_size=3 и pg_minsize=1, то часть данных может записаться всего на 1 OSD.
+    Если потом конкретно этот OSD упадёт, PG окажется **incomplete**.
+  - [allow_net_split](../config/osd.ru.md#allow_net_split) отключено (по умолчанию) и
+    первичный OSD данной PG не может соединиться с частью вторичных OSD этой PG, помеченных
+    как живых в etcd. Это означает, что произошло разделение сети: OSD могут общаться с etcd,
+    но не могут общаться с частью других OSD.
 - **offline** — PG вообще не активирована ни одним OSD. Либо первичный OSD не назначен вообще
   (если пул только создан), либо в качестве первичного назначен недоступный OSD, либо
   назначенный OSD отказывается запускать эту PG (например, из-за несовпадения block_size),

docs/usage/cli.en.md

@@ -378,11 +378,11 @@ Examples:
 
 Create a pool. Required parameters:
 
-| <!-- -->                 | <!-- -->                                                                              |
-|--------------------------|---------------------------------------------------------------------------------------|
-| `-s R` or `--pg_size R`  | Number of replicas for replicated pools                                               |
-| `--ec N+K`               | Number of data (N) and parity (K) chunks for erasure-coded pools                      |
-| `-n N` or `--pg_count N` | PG count for the new pool (start with 10*<OSD count>/pg_size rounded to a power of 2) |
+| <!-- -->                 | <!-- -->                                                                                |
+|--------------------------|-----------------------------------------------------------------------------------------|
+| `-s R` or `--pg_size R`  | Number of replicas for replicated pools                                                 |
+| `--ec N+K`               | Number of data (N) and parity (K) chunks for erasure-coded pools                        |
+| `-n N` or `--pg_count N` | PG count for the new pool (start with 10*\<OSD count\>/pg_size rounded to a power of 2) |
 
 Optional parameters:
 

docs/usage/cli.ru.md

@@ -22,6 +22,8 @@ vitastor-cli - интерфейс командной строки для адм
 - [flatten](#flatten)
 - [rm-data](#rm-data)
 - [merge-data](#merge-data)
+- [describe](#describe)
+- [fix](#fix)
 - [alloc-osd](#alloc-osd)
 - [rm-osd](#rm-osd)
 - [osd-tree](#osd-tree)
@@ -393,11 +395,11 @@ OSD  PARENT            UP    SIZE  USED%    TAGS          WEIGHT  BLOCK  BITMAP
 
 Создать пул. Обязательные параметры:
 
-| <!-- -->                  | <!-- -->                                                                                    |
-|---------------------------|---------------------------------------------------------------------------------------------|
-| `-s R` или `--pg_size R`  | Число копий данных для реплицированных пулов                                                |
-| `--ec N+K`                | Число частей данных (N) и чётности (K) для пулов с кодами коррекции ошибок                  |
-| `-n N` или `--pg_count N` | Число PG для нового пула (начните с 10*<число OSD>/pg_size, округлённого до степени двойки) |
+| <!-- -->                  | <!-- -->                                                                                      |
+|---------------------------|-----------------------------------------------------------------------------------------------|
+| `-s R` или `--pg_size R`  | Число копий данных для реплицированных пулов                                                  |
+| `--ec N+K`                | Число частей данных (N) и чётности (K) для пулов с кодами коррекции ошибок                    |
+| `-n N` или `--pg_count N` | Число PG для нового пула (начните с 10*\<число OSD\>/pg_size, округлённого до степени двойки) |
 
 Необязательные параметры:
 

mon/lp_optimizer/dsl_pgs.js

@@ -342,7 +342,7 @@ function filter_tree_by_rules(osd_tree, rules, selected)
 // Convert from
 // node_list = { id: string|number, level: string, size?: number, parent?: string|number }[]
 // to
-// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node_id[], levels: { [level]: id, ... } } }
+// node_tree = { [node_id]: { id, level, size?, parent?, children?: child_node[], levels: { [level]: id, ... } } }
 function index_tree(node_list)
 {
     const tree = { '': { children: [], levels: {} } };
@@ -357,7 +357,7 @@ function index_tree(node_list)
         tree[parent_id].children = tree[parent_id].children || [];
         tree[parent_id].children.push(tree[node.id]);
     }
-    const cur = tree[''].children;
+    const cur = [ ...tree[''].children ];
     for (let i = 0; i < cur.length; i++)
     {
         cur[i].levels[cur[i].level] = cur[i].id;

mon/osd_tree.js

@@ -15,7 +15,7 @@ function get_osd_tree(global_config, state)
         const stat = state.osd.stats[osd_num];
         const osd_cfg = state.config.osd[osd_num];
         let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
-        if (reweight < 0 || isNaN(reweight))
+        if (isNaN(reweight) || reweight < 0 || reweight > 0)
             reweight = 1;
         if (stat && stat.size && reweight && (state.osd.state[osd_num] || Number(stat.time) >= down_time ||
             osd_cfg && osd_cfg.noout))

mon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor-mon",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Vitastor SDS monitor service",
   "main": "mon-main.js",
   "scripts": {
@@ -19,6 +19,6 @@
     "eslint-plugin-node": "^11.1.0"
   },
   "engines": {
-    "node": ">=12.0.0"
+    "node": ">=12.1.0"
   }
 }

mon/scripts/make-etcd

@@ -40,6 +40,11 @@ async function run()
         console.log("/etc/systemd/system/vitastor-etcd.service already exists");
         process.exit(1);
     }
+    if (!in_docker && fs.existsSync("/etc/systemd/system/etcd.service"))
+    {
+        console.log("/etc/systemd/system/etcd.service already exists");
+        process.exit(1);
+    }
     const config = JSON.parse(fs.readFileSync(config_path, { encoding: 'utf-8' }));
     if (!config.etcd_address)
     {
@@ -97,8 +102,8 @@ WantedBy=multi-user.target
 `);
     await system(`useradd etcd`);
     await system(`systemctl daemon-reload`);
-    await system(`systemctl enable etcd`);
-    await system(`systemctl start etcd`);
+    await system(`systemctl enable vitastor-etcd`);
+    await system(`systemctl start vitastor-etcd`);
     process.exit(0);
 }
 

node-binding/package.json

@@ -1,6 +1,6 @@
 {
   "name": "vitastor",
-  "version": "2.0.0",
+  "version": "2.1.0",
   "description": "Low-level native bindings to Vitastor client library",
   "main": "index.js",
   "keywords": [

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VITASTOR_VERSION = '2.0.0'
+VITASTOR_VERSION = '2.1.0'
 
 LOG = logging.getLogger(__name__)
 

patches/pve-qemu-9.2-vitastor.patch

@@ -0,0 +1,172 @@
+Index: pve-qemu-kvm-9.2.0/block/meson.build
+===================================================================
+--- pve-qemu-kvm-9.2.0.orig/block/meson.build
++++ pve-qemu-kvm-9.2.0/block/meson.build
+@@ -126,6 +126,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+Index: pve-qemu-kvm-9.2.0/meson.build
+===================================================================
+--- pve-qemu-kvm-9.2.0.orig/meson.build
++++ pve-qemu-kvm-9.2.0/meson.build
+@@ -1590,6 +1590,26 @@ if not get_option('rbd').auto() or have_
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2478,6 +2498,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4789,6 +4810,7 @@ summary_info += {'fdt support':       fd
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+Index: pve-qemu-kvm-9.2.0/meson_options.txt
+===================================================================
+--- pve-qemu-kvm-9.2.0.orig/meson_options.txt
++++ pve-qemu-kvm-9.2.0/meson_options.txt
+@@ -200,6 +200,8 @@ option('lzo', type : 'feature', value :
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+Index: pve-qemu-kvm-9.2.0/qapi/block-core.json
+===================================================================
+--- pve-qemu-kvm-9.2.0.orig/qapi/block-core.json
++++ pve-qemu-kvm-9.2.0/qapi/block-core.json
+@@ -3481,7 +3481,7 @@
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+             'pbs',
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4592,6 +4592,28 @@
+             '*server': ['InetSocketAddressBase'] } }
+ 
+ ##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
++##
+ # @ReplicationMode:
+ #
+ # An enumeration of replication modes.
+@@ -5054,6 +5076,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5501,6 +5524,20 @@
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
+ ##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
++##
+ # @BlockdevVmdkSubformat:
+ #
+ # Subformat options for VMDK images
+@@ -5722,6 +5759,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+Index: pve-qemu-kvm-9.2.0/scripts/meson-buildoptions.sh
+===================================================================
+--- pve-qemu-kvm-9.2.0.orig/scripts/meson-buildoptions.sh
++++ pve-qemu-kvm-9.2.0/scripts/meson-buildoptions.sh
+@@ -174,6 +174,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -455,6 +456,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

patches/qemu-9.2-vitastor.patch

@@ -0,0 +1,172 @@
+diff --git a/block/meson.build b/block/meson.build
+index f1262ec2ba..3cf3e23f16 100644
+--- a/block/meson.build
++++ b/block/meson.build
+@@ -114,6 +114,7 @@ foreach m : [
+   [libnfs, 'nfs', files('nfs.c')],
+   [libssh, 'ssh', files('ssh.c')],
+   [rbd, 'rbd', files('rbd.c')],
++  [vitastor, 'vitastor', files('vitastor.c')],
+ ]
+   if m[0].found()
+     module_ss = ss.source_set()
+diff --git a/meson.build b/meson.build
+index 147097c652..2486b3aeb5 100644
+--- a/meson.build
++++ b/meson.build
+@@ -1590,6 +1590,26 @@ if not get_option('rbd').auto() or have_block
+   endif
+ endif
+ 
++vitastor = not_found
++if not get_option('vitastor').auto() or have_block
++  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
++    required: get_option('vitastor'))
++  if libvitastor_client.found()
++    if cc.links('''
++      #include <vitastor_c.h>
++      int main(void) {
++        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
++        return 0;
++      }''', dependencies: libvitastor_client)
++      vitastor = declare_dependency(dependencies: libvitastor_client)
++    elif get_option('vitastor').enabled()
++      error('could not link libvitastor_client')
++    else
++      warning('could not link libvitastor_client, disabling')
++    endif
++  endif
++endif
++
+ glusterfs = not_found
+ glusterfs_ftruncate_has_stat = false
+ glusterfs_iocb_has_stat = false
+@@ -2474,6 +2494,7 @@ endif
+ config_host_data.set('CONFIG_OPENGL', opengl.found())
+ config_host_data.set('CONFIG_PLUGIN', get_option('plugins'))
+ config_host_data.set('CONFIG_RBD', rbd.found())
++config_host_data.set('CONFIG_VITASTOR', vitastor.found())
+ config_host_data.set('CONFIG_RDMA', rdma.found())
+ config_host_data.set('CONFIG_RELOCATABLE', get_option('relocatable'))
+ config_host_data.set('CONFIG_SAFESTACK', get_option('safe_stack'))
+@@ -4778,6 +4799,7 @@ summary_info += {'fdt support':       fdt_opt == 'internal' ? 'internal' : fdt}
+ summary_info += {'libcap-ng support': libcap_ng}
+ summary_info += {'bpf support':       libbpf}
+ summary_info += {'rbd support':       rbd}
++summary_info += {'vitastor support':  vitastor}
+ summary_info += {'smartcard support': cacard}
+ summary_info += {'U2F support':       u2f}
+ summary_info += {'libusb':            libusb}
+diff --git a/meson_options.txt b/meson_options.txt
+index 5eeaf3eee5..b04eda29f9 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -200,6 +200,8 @@ option('lzo', type : 'feature', value : 'auto',
+        description: 'lzo compression support')
+ option('rbd', type : 'feature', value : 'auto',
+        description: 'Ceph block device driver')
++option('vitastor', type : 'feature', value : 'auto',
++       description: 'Vitastor block device driver')
+ option('opengl', type : 'feature', value : 'auto',
+        description: 'OpenGL support')
+ option('rdma', type : 'feature', value : 'auto',
+diff --git a/qapi/block-core.json b/qapi/block-core.json
+index fd3bcc1c17..41571ac3f9 100644
+--- a/qapi/block-core.json
++++ b/qapi/block-core.json
+@@ -3212,7 +3212,7 @@
+             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
+             'raw', 'rbd',
+             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
+-            'ssh', 'throttle', 'vdi', 'vhdx',
++            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
+             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
+             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
+@@ -4295,6 +4295,28 @@
+             '*key-secret': 'str',
+             '*server': ['InetSocketAddressBase'] } }
+ 
++##
++# @BlockdevOptionsVitastor:
++#
++# Driver specific block device options for vitastor
++#
++# @image:       Image name
++# @inode:       Inode number
++# @pool:        Pool ID
++# @size:        Desired image size in bytes
++# @config-path: Path to Vitastor configuration
++# @etcd-host:   etcd connection address(es)
++# @etcd-prefix: etcd key/value prefix
++##
++{ 'struct': 'BlockdevOptionsVitastor',
++  'data': { '*inode': 'uint64',
++            '*pool': 'uint64',
++            '*size': 'uint64',
++            '*image': 'str',
++            '*config-path': 'str',
++            '*etcd-host': 'str',
++            '*etcd-prefix': 'str' } }
++
+ ##
+ # @ReplicationMode:
+ #
+@@ -4757,6 +4779,7 @@
+       'throttle':   'BlockdevOptionsThrottle',
+       'vdi':        'BlockdevOptionsGenericFormat',
+       'vhdx':       'BlockdevOptionsGenericFormat',
++      'vitastor':   'BlockdevOptionsVitastor',
+       'virtio-blk-vfio-pci':
+                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
+                       'if': 'CONFIG_BLKIO' },
+@@ -5198,6 +5221,20 @@
+             '*cluster-size' :   'size',
+             '*encrypt' :        'RbdEncryptionCreateOptions' } }
+ 
++##
++# @BlockdevCreateOptionsVitastor:
++#
++# Driver specific image creation options for Vitastor.
++#
++# @location: Where to store the new image file.  This location cannot
++#     point to a snapshot.
++#
++# @size: Size of the virtual disk in bytes
++##
++{ 'struct': 'BlockdevCreateOptionsVitastor',
++  'data': { 'location':         'BlockdevOptionsVitastor',
++            'size':             'size' } }
++
+ ##
+ # @BlockdevVmdkSubformat:
+ #
+@@ -5420,6 +5457,7 @@
+       'ssh':            'BlockdevCreateOptionsSsh',
+       'vdi':            'BlockdevCreateOptionsVdi',
+       'vhdx':           'BlockdevCreateOptionsVhdx',
++      'vitastor':       'BlockdevCreateOptionsVitastor',
+       'vmdk':           'BlockdevCreateOptionsVmdk',
+       'vpc':            'BlockdevCreateOptionsVpc'
+   } }
+diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
+index a8066aab03..12e650e7d4 100644
+--- a/scripts/meson-buildoptions.sh
++++ b/scripts/meson-buildoptions.sh
+@@ -174,6 +174,7 @@ meson_options_help() {
+   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
+   printf "%s\n" '  qpl             Query Processing Library support'
+   printf "%s\n" '  rbd             Ceph block device driver'
++  printf "%s\n" '  vitastor        Vitastor block device driver'
+   printf "%s\n" '  rdma            Enable RDMA-based migration'
+   printf "%s\n" '  replication     replication support'
+   printf "%s\n" '  rust            Rust support'
+@@ -455,6 +456,8 @@ _meson_option_parse() {
+     --disable-qpl) printf "%s" -Dqpl=disabled ;;
+     --enable-rbd) printf "%s" -Drbd=enabled ;;
+     --disable-rbd) printf "%s" -Drbd=disabled ;;
++    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
++    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
+     --enable-rdma) printf "%s" -Drdma=enabled ;;
+     --disable-rdma) printf "%s" -Drdma=disabled ;;
+     --enable-relocatable) printf "%s" -Drelocatable=true ;;

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.0.0
+Version:        2.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.0.0.el7.tar.gz
+Source0:        vitastor-2.1.0.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.0.0
+Version:        2.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.0.0.el8.tar.gz
+Source0:        vitastor-2.1.0.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

rpm/vitastor-el9.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        2.0.0
+Version:        2.1.0
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-2.0.0.el9.tar.gz
+Source0:        vitastor-2.1.0.el9.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel

src/CMakeLists.txt

@@ -19,7 +19,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVITASTOR_VERSION="2.0.0")
+add_definitions(-DVITASTOR_VERSION="2.1.0")
 add_definitions(-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -fno-omit-frame-pointer -I ${CMAKE_SOURCE_DIR}/src)
 add_link_options(-fno-omit-frame-pointer)
 if (${WITH_ASAN})

src/blockstore/blockstore_disk.cpp

@@ -244,7 +244,7 @@ void blockstore_disk_t::calc_lengths(bool skip_meta_check)
             if (new_doesnt_fit)
             {
                 printf("Warning: Using old metadata format without checksums because the new format"
-                    " doesn't fit into provided area (%lu bytes required, %lu bytes available)\n", meta_len, meta_area_size);
+                    " doesn't fit into provided area (%ju bytes required, %ju bytes available)\n", meta_len, meta_area_size);
             }
             clean_entry_size = clean_entry_v0_size;
             meta_len = meta_v0_len;

src/client/CMakeLists.txt

@@ -7,10 +7,14 @@ set(MSGR_RDMA "")
 if (IBVERBS_LIBRARIES)
 	set(MSGR_RDMA "msgr_rdma.cpp")
 endif (IBVERBS_LIBRARIES)
+set(MSGR_RDMACM "")
+if (RDMACM_LIBRARIES)
+	set(MSGR_RDMACM "msgr_rdmacm.cpp")
+endif (RDMACM_LIBRARIES)
 add_library(vitastor_common STATIC
 	../util/epoll_manager.cpp etcd_state_client.cpp messenger.cpp ../util/addr_util.cpp
 	msgr_stop.cpp msgr_op.cpp msgr_send.cpp msgr_receive.cpp ../util/ringloop.cpp ../../json11/json11.cpp
-	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA}
+	http_client.cpp osd_ops.cpp pg_states.cpp ../util/timerfd_manager.cpp ../util/str_util.cpp ../util/json_util.cpp ${MSGR_RDMA} ${MSGR_RDMACM}
 )
 target_link_libraries(vitastor_common pthread)
 target_compile_options(vitastor_common PUBLIC -fPIC)
@@ -28,6 +32,7 @@ target_link_libraries(vitastor_client
 	vitastor_cli
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
+	${RDMACM_LIBRARIES}
 )
 set_target_properties(vitastor_client PROPERTIES VERSION ${VITASTOR_VERSION} SOVERSION 0)
 configure_file(vitastor.pc.in vitastor.pc @ONLY)

src/client/cluster_client.cpp

@@ -29,8 +29,7 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
         if (msgr.osd_peer_fds.find(peer_osd) != msgr.osd_peer_fds.end())
         {
             // peer_osd just connected
-            // retry operations waiting for connection immediately
-            continue_ops(client_retry_interval);
+            continue_ops();
             continue_lists();
             continue_raw_ops(peer_osd);
         }

src/client/cluster_client.h

@@ -83,6 +83,9 @@ class writeback_cache_t;
 // FIXME: Split into public and private interfaces
 class cluster_client_t
 {
+#ifdef __MOCK__
+public:
+#endif
     timerfd_manager_t *tfd;
     ring_loop_t *ringloop;
 
@@ -144,8 +147,6 @@ public:
 
     bool get_immediate_commit(uint64_t inode);
 
-    void continue_ops(int time_passed = 0);
-
     void list_inode(inode_t inode, uint64_t min_offset, uint64_t max_offset, int max_parallel_pgs, std::function<void(
         int status, int pgs_left, pg_num_t pg_num, std::set<object_id>&& objects)> pg_callback);
 
@@ -153,6 +154,11 @@ public:
     //inline uint64_t get_bs_block_size() { return st_cli.global_block_size; }
     uint64_t next_op_id();
 
+#ifndef __MOCK__
+protected:
+#endif
+    void continue_ops(int time_passed = 0);
+
 protected:
     bool affects_osd(uint64_t inode, uint64_t offset, uint64_t len, osd_num_t osd);
     bool affects_pg(uint64_t inode, uint64_t offset, uint64_t len, pool_id_t pool_id, pg_num_t pg_num);

src/client/etcd_state_client.cpp

@@ -31,6 +31,11 @@ etcd_state_client_t::~etcd_state_client_t()
         keepalive_client = NULL;
     }
 #endif
+    if (load_pgs_timer_id >= 0)
+    {
+        tfd->clear_timer(load_pgs_timer_id);
+        load_pgs_timer_id = -1;
+    }
 }
 
 #ifndef __MOCK__
@@ -143,6 +148,7 @@ void etcd_state_client_t::etcd_call(std::string api, json11::Json payload, int t
                 }
                 if (interval > 0)
                 {
+                    // FIXME: Prevent destruction of etcd_state_client if timers or requests are active
                     tfd->set_timer(interval, false, [this, api, payload, timeout, retries, interval, callback](int)
                     {
                         etcd_call(api, payload, timeout, retries, interval, callback);
@@ -176,7 +182,7 @@ void etcd_state_client_t::add_etcd_url(std::string addr)
             exit(1);
         }
         if (!local_ips.size())
-            local_ips = getifaddr_list(std::vector<std::string>(), true);
+            local_ips = getifaddr_list(std::vector<addr_mask_t>(), true);
         std::string check_addr;
         int pos = addr.find('/');
         int pos2 = addr.find(':');
@@ -271,6 +277,11 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
     {
         this->etcd_quick_timeout = 1000;
     }
+    this->etcd_min_reload_interval = config["etcd_min_reload_interval"].uint64_value();
+    if (this->etcd_min_reload_interval <= 0)
+    {
+        this->etcd_min_reload_interval = 50;
+    }
     if (this->etcd_ws_keepalive_interval != old_etcd_ws_keepalive_interval && ws_keepalive_timer >= 0)
     {
 #ifndef __MOCK__
@@ -603,6 +614,23 @@ void etcd_state_client_t::load_global_config()
 
 void etcd_state_client_t::load_pgs()
 {
+    timespec tv;
+    clock_gettime(CLOCK_REALTIME, &tv);
+    uint64_t ms_passed = (tv.tv_sec-etcd_last_reload.tv_sec)*1000 + (tv.tv_nsec-etcd_last_reload.tv_nsec)/1000000;
+    if (ms_passed < etcd_min_reload_interval)
+    {
+        if (load_pgs_timer_id < 0)
+        {
+            load_pgs_timer_id = tfd->set_timer(etcd_min_reload_interval+50-ms_passed, false, [this](int) { load_pgs(); });
+        }
+        return;
+    }
+    etcd_last_reload = tv;
+    if (load_pgs_timer_id >= 0)
+    {
+        tfd->clear_timer(load_pgs_timer_id);
+        load_pgs_timer_id = -1;
+    }
     json11::Json::array txn = {
         json11::Json::object {
             { "request_range", json11::Json::object {

src/client/etcd_state_client.h

@@ -108,6 +108,7 @@ public:
     int max_etcd_attempts = 5;
     int etcd_quick_timeout = 1000;
     int etcd_slow_timeout = 5000;
+    int etcd_min_reload_interval = 1000;
     bool infinite_start = true;
     uint64_t global_block_size = DEFAULT_BLOCK_SIZE;
     uint32_t global_bitmap_granularity = DEFAULT_BITMAP_GRANULARITY;
@@ -122,6 +123,8 @@ public:
     uint64_t etcd_watch_revision_config = 0;
     uint64_t etcd_watch_revision_osd = 0;
     uint64_t etcd_watch_revision_pg = 0;
+    timespec etcd_last_reload = {};
+    int load_pgs_timer_id = -1;
     std::map<pool_id_t, pool_config_t> pool_config;
     std::map<osd_num_t, json11::Json> peer_states;
     std::set<osd_num_t> seen_peers;

src/client/fio_cluster.cpp

@@ -377,7 +377,7 @@ static void io_callback(void *opaque, long retval)
     bsd->completed.push_back(io);
     if (bsd->trace)
     {
-        printf("--- %s 0x%jx retval=%ld\n", io->ddir == DDIR_READ ? "READ" :
+        printf("--- %s 0x%jx retval=%jd\n", io->ddir == DDIR_READ ? "READ" :
             (io->ddir == DDIR_WRITE ? "WRITE" : "SYNC"), (uint64_t)io, retval);
     }
 }

src/client/messenger.cpp

@@ -117,29 +117,53 @@ void msgr_iothread_t::run()
 
 void osd_messenger_t::init()
 {
+#ifdef WITH_RDMACM
+    if (use_rdmacm)
+    {
+        // RDMA-CM only requires the event channel. All the remaining work is done separately
+        rdmacm_evch = rdma_create_event_channel();
+        if (!rdmacm_evch)
+        {
+            // ENODEV means that the client doesn't have RDMA devices available
+            if (errno != ENODEV || log_level > 0)
+                fprintf(stderr, "Failed to initialize RDMA-CM event channel: %s (code %d)\n", strerror(errno), errno);
+        }
+        else
+        {
+            fcntl(rdmacm_evch->fd, F_SETFL, fcntl(rdmacm_evch->fd, F_GETFL, 0) | O_NONBLOCK);
+            tfd->set_fd_handler(rdmacm_evch->fd, false, [this](int rdmacm_eventfd, int epoll_events)
+            {
+                handle_rdmacm_events();
+            });
+        }
+    }
+    else
+#endif
 #ifdef WITH_RDMA
     if (use_rdma)
     {
-        rdma_context = msgr_rdma_context_t::create(
-            osd_networks, rdma_device != "" ? rdma_device.c_str() : NULL,
+        rdma_contexts = msgr_rdma_context_t::create_all(
+            osd_num && osd_cluster_network_masks.size() ? osd_cluster_network_masks : osd_network_masks,
+            rdma_device != "" ? rdma_device.c_str() : NULL,
             rdma_port_num, rdma_gid_index, rdma_mtu, rdma_odp, log_level
         );
-        if (!rdma_context)
+        if (!rdma_contexts.size())
         {
             if (log_level > 0)
                 fprintf(stderr, "[OSD %ju] Couldn't initialize RDMA, proceeding with TCP only\n", osd_num);
         }
         else
         {
-            rdma_max_sge = rdma_max_sge < rdma_context->attrx.orig_attr.max_sge
-                ? rdma_max_sge : rdma_context->attrx.orig_attr.max_sge;
             fprintf(stderr, "[OSD %ju] RDMA initialized successfully\n", osd_num);
-            fcntl(rdma_context->channel->fd, F_SETFL, fcntl(rdma_context->channel->fd, F_GETFL, 0) | O_NONBLOCK);
-            tfd->set_fd_handler(rdma_context->channel->fd, false, [this](int notify_fd, int epoll_events)
+            for (msgr_rdma_context_t* rdma_context: rdma_contexts)
             {
-                handle_rdma_events();
-            });
-            handle_rdma_events();
+                fcntl(rdma_context->channel->fd, F_SETFL, fcntl(rdma_context->channel->fd, F_GETFL, 0) | O_NONBLOCK);
+                tfd->set_fd_handler(rdma_context->channel->fd, false, [this, rdma_context](int notify_fd, int epoll_events)
+                {
+                    handle_rdma_events(rdma_context);
+                });
+                handle_rdma_events(rdma_context);
+            }
         }
     }
 #endif
@@ -247,10 +271,19 @@ osd_messenger_t::~osd_messenger_t()
         iothreads.clear();
     }
 #ifdef WITH_RDMA
-    if (rdma_context)
+    for (auto rdma_context: rdma_contexts)
     {
         delete rdma_context;
     }
+    rdma_contexts.clear();
+#endif
+#ifdef WITH_RDMACM
+    if (rdmacm_evch)
+    {
+        tfd->set_fd_handler(rdmacm_evch->fd, false, NULL);
+        rdma_destroy_event_channel(rdmacm_evch);
+        rdmacm_evch = NULL;
+    }
 #endif
 }
 
@@ -262,10 +295,14 @@ void osd_messenger_t::parse_config(const json11::Json & config)
         // RDMA is on by default in RDMA-enabled builds
         this->use_rdma = config["use_rdma"].bool_value() || config["use_rdma"].uint64_value() != 0;
     }
+#ifdef WITH_RDMACM
+    // Use RDMA CM? (required for iWARP and may be useful for IB)
+    // FIXME: Only parse during start
+    this->use_rdmacm = config["use_rdmacm"].bool_value() || config["use_rdmacm"].uint64_value() != 0;
+    this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
+#endif
     this->rdma_device = config["rdma_device"].string_value();
     this->rdma_port_num = (uint8_t)config["rdma_port_num"].uint64_value();
-    if (!this->rdma_port_num)
-        this->rdma_port_num = 1;
     if (!config["rdma_gid_index"].is_null())
         this->rdma_gid_index = (uint8_t)config["rdma_gid_index"].uint64_value();
     this->rdma_mtu = (uint32_t)config["rdma_mtu"].uint64_value();
@@ -282,15 +319,6 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
     this->rdma_odp = config["rdma_odp"].bool_value();
-    std::vector<std::string> mask;
-    if (config["bind_address"].is_string())
-        mask.push_back(config["bind_address"].string_value());
-    else if (config["osd_network"].is_string())
-        mask.push_back(config["osd_network"].string_value());
-    else
-        for (auto v: config["osd_network"].array_items())
-            mask.push_back(v.string_value());
-    this->osd_networks = mask;
 #endif
     if (!osd_num)
         this->iothread_count = (uint32_t)config["client_iothread_count"].uint64_value();
@@ -314,23 +342,87 @@ void osd_messenger_t::parse_config(const json11::Json & config)
     if (!this->osd_ping_timeout)
         this->osd_ping_timeout = 5;
     this->log_level = config["log_level"].uint64_value();
+    // OSD public & cluster networks
+    this->osd_networks.clear();
+    if (config["osd_network"].is_string())
+        this->osd_networks.push_back(config["osd_network"].string_value());
+    else
+        for (auto v: config["osd_network"].array_items())
+            this->osd_networks.push_back(v.string_value());
+    this->osd_cluster_networks.clear();
+    if (config["osd_cluster_network"].is_string())
+        this->osd_cluster_networks.push_back(config["osd_cluster_network"].string_value());
+    else
+        for (auto v: config["osd_cluster_network"].array_items())
+            this->osd_cluster_networks.push_back(v.string_value());
+    if (this->osd_cluster_networks.size())
+        for (auto & net: this->osd_cluster_networks)
+            for (int i = this->osd_networks.size()-1; i >= 0; i--)
+                if (this->osd_networks[i] == net)
+                    this->osd_networks.erase(this->osd_networks.begin()+i, this->osd_networks.begin()+i+1);
+    this->osd_network_masks.clear();
+    for (auto & netstr: this->osd_networks)
+        this->osd_network_masks.push_back(cidr_parse(netstr));
+    this->osd_cluster_network_masks.clear();
+    for (auto & netstr: this->osd_cluster_networks)
+        this->osd_cluster_network_masks.push_back(cidr_parse(netstr));
+    this->all_osd_networks.clear();
+    this->all_osd_networks.insert(this->all_osd_networks.end(), this->osd_networks.begin(), this->osd_networks.end());
+    this->all_osd_networks.insert(this->all_osd_networks.end(), this->osd_cluster_networks.begin(), this->osd_cluster_networks.end());
+    this->all_osd_network_masks.clear();
+    this->all_osd_network_masks.insert(this->all_osd_network_masks.end(), this->osd_network_masks.begin(), this->osd_network_masks.end());
+    this->all_osd_network_masks.insert(this->all_osd_network_masks.end(), this->osd_cluster_network_masks.begin(), this->osd_cluster_network_masks.end());
+    if (!this->osd_networks.size())
+    {
+        this->osd_networks = this->osd_cluster_networks;
+        this->osd_network_masks = this->osd_cluster_network_masks;
+    }
 }
 
 void osd_messenger_t::connect_peer(uint64_t peer_osd, json11::Json peer_state)
 {
-    if (wanted_peers.find(peer_osd) == wanted_peers.end())
+    if (wanted_peers[peer_osd].raw_address_list != peer_state["addresses"])
     {
-        wanted_peers[peer_osd] = (osd_wanted_peer_t){
-            .address_list = peer_state["addresses"],
-            .port = (int)peer_state["port"].int64_value(),
-        };
+        wanted_peers[peer_osd].raw_address_list = peer_state["addresses"];
+        // We are an OSD -> try to select a cluster address
+        // We are a client -> try to select a public address
+        // OSD only has 1 address -> don't try anything, it's pointless
+        // FIXME: Maybe support optional fallback from cluster to public network?
+        auto & match_masks = (this->osd_num ? osd_cluster_network_masks : osd_network_masks);
+        if (peer_state["addresses"].array_items().size() > 1 && match_masks.size())
+        {
+            json11::Json::array address_list;
+            for (auto json_addr: peer_state["addresses"].array_items())
+            {
+                struct sockaddr_storage addr;
+                auto ok = string_to_addr(json_addr.string_value(), false, 0, &addr);
+                if (ok)
+                {
+                    bool matches = false;
+                    for (auto & mask: match_masks)
+                    {
+                        if (cidr_sockaddr_match(addr, mask))
+                        {
+                            matches = true;
+                            break;
+                        }
+                    }
+                    if (matches)
+                        address_list.push_back(json_addr);
+                }
+            }
+            if (!address_list.size())
+                address_list = peer_state["addresses"].array_items();
+            wanted_peers[peer_osd].address_list = address_list;
+        }
+        else
+            wanted_peers[peer_osd].address_list = peer_state["addresses"];
+        wanted_peers[peer_osd].address_changed = true;
     }
-    else
-    {
-        wanted_peers[peer_osd].address_list = peer_state["addresses"];
-        wanted_peers[peer_osd].port = (int)peer_state["port"].int64_value();
-    }
-    wanted_peers[peer_osd].address_changed = true;
+#ifdef WITH_RDMACM
+    wanted_peers[peer_osd].rdmacm_port = (int)peer_state["rdmacm_port"].int64_value();
+#endif
+    wanted_peers[peer_osd].port = (int)peer_state["port"].int64_value();
     try_connect_peer(peer_osd);
 }
 
@@ -355,12 +447,24 @@ void osd_messenger_t::try_connect_peer(uint64_t peer_osd)
     wp.cur_addr = wp.address_list[wp.address_index].string_value();
     wp.cur_port = wp.port;
     wp.connecting = true;
-    try_connect_peer_addr(peer_osd, wp.cur_addr.c_str(), wp.cur_port);
+#ifdef WITH_RDMACM
+    if (use_rdmacm && wp.rdmacm_port)
+        rdmacm_try_connect_peer(peer_osd, wp.cur_addr.c_str(), wp.rdmacm_port, wp.cur_port);
+    else
+#endif
+        try_connect_peer_tcp(peer_osd, wp.cur_addr.c_str(), wp.cur_port);
 }
 
-void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port)
+void osd_messenger_t::try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port)
 {
     assert(peer_osd != this->osd_num);
+#ifdef WITH_RDMACM
+    if (disable_tcp)
+    {
+        on_connect_peer(peer_osd, -EINVAL);
+        return;
+    }
+#endif
     struct sockaddr_storage addr;
     if (!string_to_addr(peer_host, 0, peer_port, &addr))
     {
@@ -524,20 +628,30 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         },
     };
 #ifdef WITH_RDMA
-    if (rdma_context)
+    if (!use_rdmacm && rdma_contexts.size())
     {
-        cl->rdma_conn = msgr_rdma_connection_t::create(rdma_context, rdma_max_send, rdma_max_recv, rdma_max_sge, rdma_max_msg);
-        if (cl->rdma_conn)
+        // Choose the right context for the selected network
+        msgr_rdma_context_t *selected_ctx = choose_rdma_context(cl);
+        if (!selected_ctx)
         {
-            json11::Json payload = json11::Json::object {
-                { "connect_rdma", cl->rdma_conn->addr.to_string() },
-                { "rdma_max_msg", cl->rdma_conn->max_msg },
-            };
-            std::string payload_str = payload.dump();
-            op->req.show_conf.json_len = payload_str.size();
-            op->buf = malloc_or_die(payload_str.size());
-            op->iov.push_back(op->buf, payload_str.size());
-            memcpy(op->buf, payload_str.c_str(), payload_str.size());
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA context for OSD %ju connection (peer %d), using only TCP\n", cl->osd_num, cl->peer_fd);
+        }
+        else
+        {
+            cl->rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, rdma_max_msg);
+            if (cl->rdma_conn)
+            {
+                json11::Json payload = json11::Json::object {
+                    { "connect_rdma", cl->rdma_conn->addr.to_string() },
+                    { "rdma_max_msg", cl->rdma_conn->max_msg },
+                };
+                std::string payload_str = payload.dump();
+                op->req.show_conf.json_len = payload_str.size();
+                op->buf = malloc_or_die(payload_str.size());
+                op->iov.push_back(op->buf, payload_str.size());
+                memcpy(op->buf, payload_str.c_str(), payload_str.size());
+            }
         }
     }
 #endif
@@ -582,29 +696,23 @@ void osd_messenger_t::check_peer_config(osd_client_t *cl)
         {
             osd_num_t peer_osd = cl->osd_num;
             stop_client(op->peer_fd);
-            on_connect_peer(peer_osd, -1);
+            on_connect_peer(peer_osd, -EINVAL);
             delete op;
             return;
         }
 #ifdef WITH_RDMA
-        if (config["rdma_address"].is_string())
+        if (!use_rdmacm && cl->rdma_conn && config["rdma_address"].is_string())
         {
             msgr_rdma_address_t addr;
             if (!msgr_rdma_address_t::from_string(config["rdma_address"].string_value().c_str(), &addr) ||
                 cl->rdma_conn->connect(&addr) != 0)
             {
                 fprintf(
-                    stderr, "Failed to connect to OSD %ju (address %s) using RDMA\n",
+                    stderr, "Failed to connect to OSD %ju (address %s) using RDMA, switching back to TCP\n",
                     cl->osd_num, config["rdma_address"].string_value().c_str()
                 );
                 delete cl->rdma_conn;
                 cl->rdma_conn = NULL;
-                // FIXME: Keep TCP connection in this case
-                osd_num_t peer_osd = cl->osd_num;
-                stop_client(cl->peer_fd);
-                on_connect_peer(peer_osd, -1);
-                delete op;
-                return;
             }
             else
             {
@@ -673,9 +781,30 @@ void osd_messenger_t::accept_connections(int listen_fd)
 }
 
 #ifdef WITH_RDMA
+msgr_rdma_context_t* osd_messenger_t::choose_rdma_context(osd_client_t *cl)
+{
+    // Choose the right context for the selected network
+    msgr_rdma_context_t *selected_ctx = NULL;
+    for (auto rdma_ctx: rdma_contexts)
+    {
+        if (!rdma_ctx->net_mask.family && !selected_ctx ||
+            rdma_ctx->net_mask.family && cidr_sockaddr_match(cl->peer_addr, rdma_ctx->net_mask))
+        {
+            selected_ctx = rdma_ctx;
+        }
+    }
+    return selected_ctx;
+}
+
 bool osd_messenger_t::is_rdma_enabled()
 {
-    return rdma_context != NULL;
+    return rdma_contexts.size() > 0;
+}
+#endif
+#ifdef WITH_RDMACM
+bool osd_messenger_t::is_use_rdmacm()
+{
+    return use_rdmacm;
 }
 #endif
 

src/client/messenger.h

@@ -16,6 +16,7 @@
 #include "json11/json11.hpp"
 #include "msgr_op.h"
 #include "timerfd_manager.h"
+#include "addr_util.h"
 #include <ringloop.h>
 
 #define CL_READ_HDR 1
@@ -49,10 +50,10 @@ struct osd_client_t
 {
     int refs = 0;
 
-    sockaddr_storage peer_addr;
-    int peer_port;
+    sockaddr_storage peer_addr = {};
+    int peer_port = 0;
     int peer_fd = -1;
-    int peer_state;
+    int peer_state = 0;
     int connect_timeout_id = -1;
     int ping_time_remaining = 0;
     int idle_time_remaining = 0;
@@ -93,13 +94,18 @@ struct osd_client_t
 
 struct osd_wanted_peer_t
 {
+    json11::Json raw_address_list;
     json11::Json address_list;
-    int port;
-    time_t last_connect_attempt;
-    bool connecting, address_changed;
-    int address_index;
+    int port = 0;
+    // FIXME: Remove separate WITH_RDMACM?
+#ifdef WITH_RDMACM
+    int rdmacm_port = 0;
+#endif
+    time_t last_connect_attempt = 0;
+    bool connecting = false, address_changed = false;
+    int address_index = 0;
     std::string cur_addr;
-    int cur_port;
+    int cur_port = 0;
 };
 
 struct osd_op_stats_t
@@ -149,6 +155,15 @@ public:
 };
 #endif
 
+#ifdef WITH_RDMA
+struct rdma_event_channel;
+struct rdma_cm_id;
+struct rdma_cm_event;
+struct ibv_context;
+struct osd_messenger_t;
+struct rdmacm_connecting_t;
+#endif
+
 struct osd_messenger_t
 {
 protected:
@@ -165,14 +180,19 @@ protected:
 
 #ifdef WITH_RDMA
     bool use_rdma = true;
-    std::vector<std::string> osd_networks;
+    bool use_rdmacm = false;
+    bool disable_tcp = false;
     std::string rdma_device;
-    uint64_t rdma_port_num = 1, rdma_mtu = 0;
+    uint64_t rdma_port_num = 1;
+    int rdma_mtu = 0;
     int rdma_gid_index = -1;
-    msgr_rdma_context_t *rdma_context = NULL;
+    std::vector<msgr_rdma_context_t *> rdma_contexts;
     uint64_t rdma_max_sge = 0, rdma_max_send = 0, rdma_max_recv = 0;
     uint64_t rdma_max_msg = 0;
     bool rdma_odp = false;
+    rdma_event_channel *rdmacm_evch = NULL;
+    std::map<rdma_cm_id*, osd_client_t*> rdmacm_connections;
+    std::map<rdma_cm_id*, rdmacm_connecting_t*> rdmacm_connecting;
 #endif
 
     std::vector<msgr_iothread_t*> iothreads;
@@ -190,6 +210,12 @@ public:
     std::map<int, osd_client_t*> clients;
     std::map<osd_num_t, osd_wanted_peer_t> wanted_peers;
     std::map<uint64_t, int> osd_peer_fds;
+    std::vector<std::string> osd_networks;
+    std::vector<addr_mask_t> osd_network_masks;
+    std::vector<std::string> osd_cluster_networks;
+    std::vector<addr_mask_t> osd_cluster_network_masks;
+    std::vector<std::string> all_osd_networks;
+    std::vector<addr_mask_t> all_osd_network_masks;
     // op statistics
     osd_op_stats_t stats, recovery_stats;
 
@@ -216,13 +242,18 @@ public:
     bool is_rdma_enabled();
     bool connect_rdma(int peer_fd, std::string rdma_address, uint64_t client_max_msg);
 #endif
+#ifdef WITH_RDMACM
+    bool is_use_rdmacm();
+    rdma_cm_id *rdmacm_listen(const std::string & bind_address, int rdmacm_port, int *bound_port, int log_level);
+    void rdmacm_destroy_listener(rdma_cm_id *listener);
+#endif
 
     void inc_op_stats(osd_op_stats_t & stats, uint64_t opcode, timespec & tv_begin, timespec & tv_end, uint64_t len);
     void measure_exec(osd_op_t *cur_op);
 
 protected:
     void try_connect_peer(uint64_t osd_num);
-    void try_connect_peer_addr(osd_num_t peer_osd, const char *peer_host, int peer_port);
+    void try_connect_peer_tcp(osd_num_t peer_osd, const char *peer_host, int peer_port);
     void handle_peer_epoll(int peer_fd, int epoll_events);
     void handle_connect_epoll(int peer_fd);
     void on_connect_peer(osd_num_t peer_osd, int peer_fd);
@@ -247,6 +278,19 @@ protected:
     void try_send_rdma_odp(osd_client_t *cl);
     void try_send_rdma_nodp(osd_client_t *cl);
     bool try_recv_rdma(osd_client_t *cl);
-    void handle_rdma_events();
+    void handle_rdma_events(msgr_rdma_context_t *rdma_context);
+    msgr_rdma_context_t* choose_rdma_context(osd_client_t *cl);
+#endif
+#ifdef WITH_RDMACM
+    void handle_rdmacm_events();
+    msgr_rdma_context_t* rdmacm_get_context(ibv_context *verbs);
+    msgr_rdma_context_t* rdmacm_create_qp(rdma_cm_id *cmid);
+    void rdmacm_accept(rdma_cm_event *ev);
+    void rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int rdmacm_port, int fallback_tcp_port);
+    void rdmacm_set_conn_timeout(rdmacm_connecting_t *conn);
+    void rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res);
+    void rdmacm_address_resolved(rdma_cm_event *ev);
+    void rdmacm_route_resolved(rdma_cm_event *ev);
+    void rdmacm_established(rdma_cm_event *ev);
 #endif
 };

src/client/msgr_rdma.cpp

@@ -3,10 +3,35 @@
 
 #include <stdio.h>
 #include <stdlib.h>
-#include "addr_util.h"
 #include "msgr_rdma.h"
 #include "messenger.h"
 
+static uint32_t ibv_mtu_to_bytes(ibv_mtu mtu)
+{
+    switch (mtu)
+    {
+        case IBV_MTU_256:  return 256;
+        case IBV_MTU_512:  return 512;
+        case IBV_MTU_1024: return 1024;
+        case IBV_MTU_2048: return 2048;
+        case IBV_MTU_4096: return 4096;
+    }
+    return 4096;
+}
+
+static ibv_mtu bytes_to_ibv_mtu(uint32_t mtu)
+{
+    switch (mtu)
+    {
+        case 256:  return IBV_MTU_256;
+        case 512:  return IBV_MTU_512;
+        case 1024: return IBV_MTU_1024;
+        case 2048: return IBV_MTU_2048;
+        case 4096: return IBV_MTU_4096;
+    }
+    return IBV_MTU_4096;
+}
+
 std::string msgr_rdma_address_t::to_string()
 {
     char msg[sizeof "0000:00000000:00000000:00000000000000000000000000000000"];
@@ -38,15 +63,27 @@ msgr_rdma_context_t::~msgr_rdma_context_t()
         ibv_dereg_mr(mr);
     if (pd)
         ibv_dealloc_pd(pd);
-    if (context)
+    if (context && !is_cm)
         ibv_close_device(context);
 }
 
 msgr_rdma_connection_t::~msgr_rdma_connection_t()
 {
-    ctx->used_max_cqe -= max_send+max_recv;
+    ctx->reserve_cqe(-max_send-max_recv);
+#ifdef WITH_RDMACM
+    if (qp && !cmid)
+        ibv_destroy_qp(qp);
+    if (cmid)
+    {
+        ctx->cm_refs--;
+        if (cmid->qp)
+            rdma_destroy_qp(cmid);
+        rdma_destroy_id(cmid);
+    }
+#else
     if (qp)
         ibv_destroy_qp(qp);
+#endif
     if (recv_buffers.size())
     {
         for (auto b: recv_buffers)
@@ -77,21 +114,21 @@ static bool is_ipv4_gid(ibv_gid_entry *gidx)
         ((uint32_t*)gidx->gid.raw)[2] == 0xffff0000);
 }
 
-static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet)
+static int match_gid(ibv_gid_entry *gidx, const addr_mask_t *networks, int nnet)
 {
     if (gidx->gid_type != IBV_GID_TYPE_ROCE_V1 &&
         gidx->gid_type != IBV_GID_TYPE_ROCE_V2 ||
         ((uint64_t*)gidx->gid.raw)[0] == 0 &&
         ((uint64_t*)gidx->gid.raw)[1] == 0)
     {
-        return false;
+        return -1;
     }
     if (is_ipv4_gid(gidx))
     {
         for (int i = 0; i < nnet; i++)
         {
             if (networks[i].family == AF_INET && cidr_match(*(in_addr*)(gidx->gid.raw+12), networks[i].ipv4, networks[i].bits))
-                return true;
+                return i;
         }
     }
     else
@@ -99,119 +136,67 @@ static bool match_gid(ibv_gid_entry *gidx, addr_mask_t *networks, int nnet)
         for (int i = 0; i < nnet; i++)
         {
             if (networks[i].family == AF_INET6 && cidr6_match(*(in6_addr*)gidx->gid.raw, networks[i].ipv6, networks[i].bits))
-                return true;
+                return i;
         }
     }
-    return false;
+    return -1;
 }
 
-struct matched_dev
-{
-    int dev = -1;
-    int port = -1;
-    int gid = -1;
-    bool rocev2 = false;
-};
-
-static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, ibv_gid_entry & gidx)
+static void log_rdma_dev_port_gid(ibv_device *dev, int ib_port, int gid_index, int mtu, ibv_gid_entry & gidx)
 {
     bool is4 = ((uint64_t*)gidx.gid.raw)[0] == 0 && ((uint32_t*)gidx.gid.raw)[2] == 0xffff0000;
     char buf[256];
     inet_ntop(is4 ? AF_INET : AF_INET6, is4 ? gidx.gid.raw+12 : gidx.gid.raw, buf, sizeof(buf));
     fprintf(
-        stderr, "Auto-selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s\n",
+        stderr, "Selected RDMA device %s port %d GID %d - ROCEv%d IPv%d %s, MTU %d\n",
         ibv_get_device_name(dev), ib_port, gid_index,
-        gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf
+        gidx.gid_type == IBV_GID_TYPE_ROCE_V2 ? 2 : 1, is4 ? 4 : 6, buf, mtu
     );
 }
 
-static matched_dev match_device(ibv_device **dev_list, addr_mask_t *networks, int nnet, int log_level)
+static int match_port_gid(const std::vector<addr_mask_t> & osd_network_masks, ibv_context *context,
+    int port_num, int gid_count, int log_level, ibv_gid_entry *best_gidx, int *net_num)
 {
-    matched_dev best;
-    ibv_device_attr attr;
-    ibv_port_attr portinfo;
-    ibv_gid_entry best_gidx;
-    int res;
-    bool have_non_roce = false, have_roce = false;
-    for (int i = 0; dev_list[i]; ++i)
+    // Try to find a port with matching address
+    int best_gid_idx = -1, res = 0;
+    for (int k = 0; k < gid_count; k++)
     {
-        auto dev = dev_list[i];
-        ibv_context *context = ibv_open_device(dev_list[i]);
-        if ((res = ibv_query_device(context, &attr)) != 0)
+        ibv_gid_entry gidx;
+        if ((res = ibv_query_gid_ex(context, port_num, k, &gidx, 0)) != 0)
         {
-            fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev_list[i]), strerror(res));
-            goto cleanup;
-        }
-        for (int j = 1; j <= attr.phys_port_cnt; j++)
-        {
-            // Try to find a port with matching address
-            if ((res = ibv_query_port(context, j, &portinfo)) != 0)
+            if (res != ENODATA)
             {
-                fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), j, strerror(res));
-                goto cleanup;
+                fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(context->device), k, strerror(res));
+                continue;
             }
-            for (int k = 0; k < portinfo.gid_tbl_len; k++)
+            else
+                break;
+        }
+        if ((res = match_gid(&gidx, osd_network_masks.data(), osd_network_masks.size())) >= 0)
+        {
+            // Prefer RoCEv2
+            if (best_gid_idx < 0 || best_gidx->gid_type != IBV_GID_TYPE_ROCE_V2 && gidx.gid_type == IBV_GID_TYPE_ROCE_V2)
             {
-                ibv_gid_entry gidx;
-                if ((res = ibv_query_gid_ex(context, j, k, &gidx, 0)) != 0)
-                {
-                    if (res != ENODATA)
-                    {
-                        fprintf(stderr, "Couldn't read RDMA device %s GID index %d: %s\n", ibv_get_device_name(dev), k, strerror(res));
-                        goto cleanup;
-                    }
-                    else
-                        break;
-                }
-                if (gidx.gid_type != IBV_GID_TYPE_ROCE_V1 &&
-                    gidx.gid_type != IBV_GID_TYPE_ROCE_V2)
-                    have_non_roce = true;
-                else
-                    have_roce = true;
-                if (match_gid(&gidx, networks, nnet))
-                {
-                    // Prefer RoCEv2
-                    if (!best.rocev2)
-                    {
-                        best.dev = i;
-                        best.port = j;
-                        best.gid = k;
-                        best.rocev2 = (gidx.gid_type == IBV_GID_TYPE_ROCE_V2);
-                        best_gidx = gidx;
-                    }
-                }
+                best_gid_idx = k;
+                *best_gidx = gidx;
+                *net_num = res;
             }
         }
-cleanup:
-        ibv_close_device(context);
-        if (best.rocev2)
-        {
-            break;
-        }
     }
-    if (best.dev >= 0 && log_level > 0)
-    {
-        log_rdma_dev_port_gid(dev_list[best.dev], best.port, best.gid, best_gidx);
-    }
-    if (best.dev < 0 && have_non_roce && !have_roce)
-    {
-        best.dev = -2;
-    }
-    return best;
+    return best_gid_idx;
 }
 #endif
 
-msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
+std::vector<msgr_rdma_context_t*> msgr_rdma_context_t::create_all(const std::vector<addr_mask_t> & osd_network_masks,
+    const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level)
 {
     int res;
+    std::vector<msgr_rdma_context_t*> ret;
+    ibv_device **raw_dev_list = NULL;
     ibv_device **dev_list = NULL;
-    msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
-    ctx->mtu = mtu;
+    ibv_device *single_list[2] = {};
 
-    timespec tv;
-    clock_gettime(CLOCK_REALTIME, &tv);
-    srand48(tv.tv_sec*1000000000 + tv.tv_nsec);
-    dev_list = ibv_get_device_list(NULL);
+    raw_dev_list = dev_list = ibv_get_device_list(NULL);
     if (!dev_list || !*dev_list)
     {
         if (errno == -ENOSYS || errno == ENOSYS)
@@ -228,121 +213,131 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_ne
             fprintf(stderr, "Failed to get RDMA device list: %s\n", strerror(errno));
         goto cleanup;
     }
-    if (ib_devname)
+
+    if (sel_dev_name)
     {
         int i;
         for (i = 0; dev_list[i]; ++i)
-            if (!strcmp(ibv_get_device_name(dev_list[i]), ib_devname))
+            if (!strcmp(ibv_get_device_name(dev_list[i]), sel_dev_name))
                 break;
-        ctx->dev = dev_list[i];
-        if (!ctx->dev)
+        if (!dev_list[i])
         {
-            fprintf(stderr, "RDMA device %s not found\n", ib_devname);
+            fprintf(stderr, "RDMA device %s not found\n", sel_dev_name);
             goto cleanup;
         }
-    }
-#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
-    else if (osd_networks.size())
-    {
-        std::vector<addr_mask_t> nets;
-        for (auto & netstr: osd_networks)
-        {
-            nets.push_back(cidr_parse(netstr));
-        }
-        auto best = match_device(dev_list, nets.data(), nets.size(), log_level);
-        if (best.dev == -2)
-        {
-            best.dev = 0;
-            if (log_level > 0)
-                fprintf(stderr, "No RoCE devices found, using first available RDMA device %s\n", ibv_get_device_name(*dev_list));
-        }
-        else if (best.dev < 0)
-        {
-            if (log_level > 0)
-                fprintf(stderr, "RDMA device matching osd_network is not found, disabling RDMA\n");
-            goto cleanup;
-        }
-        else
-        {
-            ib_port = best.port;
-            gid_index = best.gid;
-        }
-        ctx->dev = dev_list[best.dev];
-    }
-#endif
-    else
-    {
-        ctx->dev = *dev_list;
+        single_list[0] = dev_list[i];
+        dev_list = single_list;
     }
 
-    ctx->context = ibv_open_device(ctx->dev);
-    if (!ctx->context)
+    for (int i = 0; dev_list[i]; ++i)
     {
-        fprintf(stderr, "Couldn't get RDMA context for %s\n", ibv_get_device_name(ctx->dev));
-        goto cleanup;
-    }
-
-    ctx->ib_port = ib_port;
-    if ((res = ibv_query_port(ctx->context, ib_port, &ctx->portinfo)) != 0)
-    {
-        fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(ctx->dev), ib_port, strerror(res));
-        goto cleanup;
-    }
-    ctx->my_lid = ctx->portinfo.lid;
-    if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && !ctx->my_lid)
-    {
-        fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(ctx->dev));
-        goto cleanup;
-    }
-
-#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
-    if (gid_index != -1)
-#endif
-    {
-        ctx->gid_index = gid_index < 0 ? 0 : gid_index;
-        if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+        auto dev = dev_list[i];
+        ibv_context *context = ibv_open_device(dev);
+        if (!context)
         {
-            fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), gid_index);
-            goto cleanup;
+            fprintf(stderr, "Couldn't get RDMA context for %s\n", ibv_get_device_name(dev));
+            continue;
         }
-    }
-#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
-    else
-    {
-        // Auto-guess GID
-        ibv_gid_entry best_gidx;
-        for (int k = 0; k < ctx->portinfo.gid_tbl_len; k++)
+        ibv_device_attr attr;
+        if ((res = ibv_query_device(context, &attr)) != 0)
         {
-            ibv_gid_entry gidx;
-            if (ibv_query_gid_ex(ctx->context, ib_port, k, &gidx, 0) != 0)
+            fprintf(stderr, "Couldn't query RDMA device %s for its features: %s\n", ibv_get_device_name(dev), strerror(res));
+            goto cleanup_dev;
+        }
+        if (sel_port_num && sel_port_num > attr.phys_port_cnt)
+        {
+            fprintf(stderr, "RDMA device %s port %d does not exist\n", ibv_get_device_name(dev), sel_port_num);
+            goto cleanup_dev;
+        }
+        for (int port_num = (sel_port_num ? sel_port_num : 1); port_num <= (sel_port_num ? sel_port_num : attr.phys_port_cnt); port_num++)
+        {
+            ibv_port_attr portinfo;
+            if ((res = ibv_query_port(context, port_num, &portinfo)) != 0)
             {
-                fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(ctx->dev), k);
-                goto cleanup;
+                fprintf(stderr, "Couldn't get RDMA device %s port %d info: %s\n", ibv_get_device_name(dev), port_num, strerror(res));
+                continue;
             }
-            // Skip empty GID
-            if (((uint64_t*)gidx.gid.raw)[0] == 0 &&
-                ((uint64_t*)gidx.gid.raw)[1] == 0)
+            if (portinfo.state != IBV_PORT_ACTIVE)
             {
                 continue;
             }
-            // Prefer IPv4 RoCEv2 -> IPv6 RoCEv2 -> IPv4 RoCEv1 -> IPv6 RoCEv1 -> IB
-            if (gid_index == -1 ||
-                gidx.gid_type == IBV_GID_TYPE_ROCE_V2 && best_gidx.gid_type != IBV_GID_TYPE_ROCE_V2 ||
-                gidx.gid_type == IBV_GID_TYPE_ROCE_V1 && best_gidx.gid_type == IBV_GID_TYPE_IB ||
-                gidx.gid_type == best_gidx.gid_type && is_ipv4_gid(&gidx))
+            if (sel_gid_index >= (int)portinfo.gid_tbl_len)
             {
-                gid_index = k;
-                best_gidx = gidx;
+                fprintf(stderr, "RDMA device %s port %d GID %d does not exist\n", ibv_get_device_name(dev), port_num, sel_gid_index);
+                continue;
+            }
+            uint32_t port_mtu = sel_mtu ? sel_mtu : ibv_mtu_to_bytes(portinfo.active_mtu);
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+            if (sel_gid_index < 0)
+            {
+                ibv_gid_entry best_gidx;
+                int net_num = 0;
+                int best_gid_idx = match_port_gid(osd_network_masks, context, port_num, portinfo.gid_tbl_len, log_level, &best_gidx, &net_num);
+                if (best_gid_idx >= 0)
+                {
+                    if (log_level > 0)
+                        log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, best_gidx);
+                    auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                    if (ctx)
+                    {
+                        ctx->net_mask = osd_network_masks[net_num];
+                        ret.push_back(ctx);
+                    }
+                }
+            }
+            else
+#endif
+            {
+                int best_gid_idx = sel_gid_index >= 0 ? sel_gid_index : 0;
+#ifdef IBV_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT
+                if (log_level > 0)
+                {
+                    ibv_gid_entry gidx;
+                    ibv_query_gid_ex(context, port_num, best_gid_idx, &gidx, 0);
+                    log_rdma_dev_port_gid(dev, port_num, best_gid_idx, port_mtu, gidx);
+                }
+#endif
+                auto ctx = msgr_rdma_context_t::create(dev, portinfo, port_num, best_gid_idx, port_mtu, odp, log_level);
+                if (ctx)
+                    ret.push_back(ctx);
             }
         }
-        ctx->gid_index = gid_index = (gid_index == -1 ? 0 : gid_index);
-        if (log_level > 0)
-        {
-            log_rdma_dev_port_gid(ctx->dev, ctx->ib_port, ctx->gid_index, best_gidx);
-        }
-        ctx->my_gid = best_gidx.gid;
+cleanup_dev:
+        ibv_close_device(context);
+    }
+
+cleanup:
+    if (raw_dev_list)
+        ibv_free_device_list(raw_dev_list);
+    return ret;
+}
+
+msgr_rdma_context_t *msgr_rdma_context_t::create(ibv_device *dev, ibv_port_attr & portinfo, int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level)
+{
+    msgr_rdma_context_t *ctx = new msgr_rdma_context_t();
+    ibv_context *context = ibv_open_device(dev);
+    if (!context)
+    {
+        fprintf(stderr, "Couldn't get RDMA context for %s\n", ibv_get_device_name(dev));
+        goto cleanup;
+    }
+
+    ctx->mtu = mtu;
+    ctx->context = context;
+    ctx->ib_port = ib_port;
+    ctx->portinfo = portinfo;
+    ctx->my_lid = ctx->portinfo.lid;
+    if (ctx->portinfo.link_layer != IBV_LINK_LAYER_ETHERNET && !ctx->my_lid)
+    {
+        fprintf(stderr, "RDMA device %s must have local LID because it's not Ethernet, but LID is zero\n", ibv_get_device_name(dev));
+        goto cleanup;
+    }
+    ctx->gid_index = gid_index;
+    if (ibv_query_gid(ctx->context, ib_port, gid_index, &ctx->my_gid))
+    {
+        fprintf(stderr, "Couldn't read RDMA device %s GID index %d\n", ibv_get_device_name(dev), gid_index);
+        goto cleanup;
     }
-#endif
 
     ctx->pd = ibv_alloc_pd(ctx->context);
     if (!ctx->pd)
@@ -351,18 +346,19 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_ne
         goto cleanup;
     }
 
+    if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
     {
-        if (ibv_query_device_ex(ctx->context, NULL, &ctx->attrx))
-        {
-            fprintf(stderr, "Couldn't query RDMA device for its features\n");
-            goto cleanup;
-        }
-        ctx->odp = odp;
-        if (ctx->odp &&
-            (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
+        fprintf(stderr, "Couldn't query RDMA device for its features\n");
+        goto cleanup;
+    }
+
+    ctx->odp = odp;
+    if (ctx->odp)
+    {
+        if (!(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT) ||
             !(ctx->attrx.odp_caps.general_caps & IBV_ODP_SUPPORT_IMPLICIT) ||
             !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_SEND) ||
-            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV)))
+            !(ctx->attrx.odp_caps.per_transport_caps.rc_odp_caps & IBV_ODP_SUPPORT_RECV))
         {
             ctx->odp = false;
             if (log_level > 0)
@@ -395,20 +391,43 @@ msgr_rdma_context_t *msgr_rdma_context_t::create(std::vector<std::string> osd_ne
         goto cleanup;
     }
 
-    if (dev_list)
-        ibv_free_device_list(dev_list);
     return ctx;
 
 cleanup:
+    if (context)
+        ibv_close_device(context);
     delete ctx;
-    if (dev_list)
-        ibv_free_device_list(dev_list);
     return NULL;
 }
 
+bool msgr_rdma_context_t::reserve_cqe(int n)
+{
+    this->used_max_cqe += n;
+    if (this->used_max_cqe > this->max_cqe)
+    {
+        // Resize CQ
+        // Mellanox ConnectX-4 supports up to 4194303 CQEs, so it's fine to put everything into a single CQ
+        int new_max_cqe = this->max_cqe;
+        while (this->used_max_cqe > new_max_cqe)
+        {
+            new_max_cqe *= 2;
+        }
+        if (ibv_resize_cq(this->cq, new_max_cqe) != 0)
+        {
+            fprintf(stderr, "Couldn't resize RDMA completion queue to %d entries\n", new_max_cqe);
+            return false;
+        }
+        this->max_cqe = new_max_cqe;
+    }
+    return true;
+}
+
 msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx, uint32_t max_send,
     uint32_t max_recv, uint32_t max_sge, uint32_t max_msg)
 {
+    if (!ctx->reserve_cqe(max_send+max_recv))
+        return NULL;
+
     msgr_rdma_connection_t *conn = new msgr_rdma_connection_t;
 
     max_sge = max_sge > ctx->attrx.orig_attr.max_sge ? ctx->attrx.orig_attr.max_sge : max_sge;
@@ -419,25 +438,6 @@ msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx,
     conn->max_sge = max_sge;
     conn->max_msg = max_msg;
 
-    ctx->used_max_cqe += max_send+max_recv;
-    if (ctx->used_max_cqe > ctx->max_cqe)
-    {
-        // Resize CQ
-        // Mellanox ConnectX-4 supports up to 4194303 CQEs, so it's fine to put everything into a single CQ
-        int new_max_cqe = ctx->max_cqe;
-        while (ctx->used_max_cqe > new_max_cqe)
-        {
-            new_max_cqe *= 2;
-        }
-        if (ibv_resize_cq(ctx->cq, new_max_cqe) != 0)
-        {
-            fprintf(stderr, "Couldn't resize RDMA completion queue to %d entries\n", new_max_cqe);
-            delete conn;
-            return NULL;
-        }
-        ctx->max_cqe = new_max_cqe;
-    }
-
     ibv_qp_init_attr init_attr = {
         .send_cq = ctx->cq,
         .recv_cq = ctx->cq,
@@ -469,9 +469,10 @@ msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx,
         .port_num        = ctx->ib_port,
     };
 
-    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS))
+    int r = 0;
+    if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | IBV_QP_PORT | IBV_QP_ACCESS_FLAGS)) != 0)
     {
-        fprintf(stderr, "Failed to switch RDMA queue pair to INIT state\n");
+        fprintf(stderr, "Failed to switch RDMA queue pair to INIT state: %s (code %d)\n", strerror(r), r);
         delete conn;
         return NULL;
     }
@@ -479,25 +480,12 @@ msgr_rdma_connection_t *msgr_rdma_connection_t::create(msgr_rdma_context_t *ctx,
     return conn;
 }
 
-static ibv_mtu mtu_to_ibv_mtu(uint32_t mtu)
-{
-    switch (mtu)
-    {
-        case 256:  return IBV_MTU_256;
-        case 512:  return IBV_MTU_512;
-        case 1024: return IBV_MTU_1024;
-        case 2048: return IBV_MTU_2048;
-        case 4096: return IBV_MTU_4096;
-    }
-    return IBV_MTU_4096;
-}
-
 int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
 {
     auto conn = this;
     ibv_qp_attr attr = {
         .qp_state       = IBV_QPS_RTR,
-        .path_mtu       = mtu_to_ibv_mtu(conn->ctx->mtu),
+        .path_mtu       = bytes_to_ibv_mtu(conn->ctx->mtu),
         .rq_psn         = dest->psn,
         .sq_psn         = conn->addr.psn,
         .dest_qp_num    = dest->qpn,
@@ -522,18 +510,19 @@ int msgr_rdma_connection_t::connect(msgr_rdma_address_t *dest)
         .rnr_retry      = 7,
     };
     // FIXME No idea if ibv_modify_qp is a blocking operation or not. No idea if it has a timeout and what it is.
-    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
-        IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER))
+    int r;
+    if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_AV | IBV_QP_PATH_MTU |
+        IBV_QP_DEST_QPN | IBV_QP_RQ_PSN | IBV_QP_MAX_DEST_RD_ATOMIC | IBV_QP_MIN_RNR_TIMER)) != 0)
     {
-        fprintf(stderr, "Failed to switch RDMA queue pair to RTR (ready-to-receive) state\n");
-        return 1;
+        fprintf(stderr, "Failed to switch RDMA queue pair to RTR (ready-to-receive) state: %s (code %d)\n", strerror(r), r);
+        return -r;
     }
     attr.qp_state = IBV_QPS_RTS;
-    if (ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_TIMEOUT |
-        IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC))
+    if ((r = ibv_modify_qp(conn->qp, &attr, IBV_QP_STATE | IBV_QP_TIMEOUT |
+        IBV_QP_RETRY_CNT | IBV_QP_RNR_RETRY | IBV_QP_SQ_PSN | IBV_QP_MAX_QP_RD_ATOMIC)) != 0)
     {
-        fprintf(stderr, "Failed to switch RDMA queue pair to RTS (ready-to-send) state\n");
-        return 1;
+        fprintf(stderr, "Failed to switch RDMA queue pair to RTS (ready-to-send) state: %s (code %d)\n", strerror(r), r);
+        return -r;
     }
     return 0;
 }
@@ -548,7 +537,15 @@ bool osd_messenger_t::connect_rdma(int peer_fd, std::string rdma_address, uint64
         {
             client_max_msg = rdma_max_msg;
         }
-        auto rdma_conn = msgr_rdma_connection_t::create(rdma_context, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
+        auto cl = clients.at(peer_fd);
+        msgr_rdma_context_t *selected_ctx = choose_rdma_context(cl);
+        if (!selected_ctx)
+        {
+            if (log_level > 0)
+                fprintf(stderr, "No RDMA context for peer %d, using only TCP\n", cl->peer_fd);
+            return false;
+        }
+        msgr_rdma_connection_t *rdma_conn = msgr_rdma_connection_t::create(selected_ctx, rdma_max_send, rdma_max_recv, rdma_max_sge, client_max_msg);
         if (rdma_conn)
         {
             int r = rdma_conn->connect(&addr);
@@ -586,7 +583,7 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
     int err = ibv_post_send(cl->rdma_conn->qp, &wr, &bad_wr);
     if (err || bad_wr)
     {
-        fprintf(stderr, "RDMA send failed: %s\n", strerror(err));
+        fprintf(stderr, "RDMA send failed: %s (code %d)\n", strerror(err), err);
         exit(1);
     }
     cl->rdma_conn->cur_send++;
@@ -667,9 +664,9 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
         // Allocate send ring buffer, if not yet
         rc->send_out_size = rc->max_msg*rdma_max_send;
         rc->send_out.buf = malloc_or_die(rc->send_out_size);
-        if (!rdma_context->odp)
+        if (!rc->ctx->odp)
         {
-            rc->send_out.mr = ibv_reg_mr(rdma_context->pd, rc->send_out.buf, rc->send_out_size, 0);
+            rc->send_out.mr = ibv_reg_mr(rc->ctx->pd, rc->send_out.buf, rc->send_out_size, 0);
             if (!rc->send_out.mr)
             {
                 fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
@@ -699,7 +696,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
             ibv_sge sge = {
                 .addr = (uintptr_t)dst,
                 .length = (uint32_t)copied,
-                .lkey = rdma_context->odp ? rdma_context->mr->lkey : rc->send_out.mr->lkey,
+                .lkey = rc->ctx->odp ? rc->ctx->mr->lkey : rc->send_out.mr->lkey,
             };
             try_send_rdma_wr(cl, &sge, 1);
             rc->send_sizes.push_back(copied);
@@ -709,7 +706,7 @@ void osd_messenger_t::try_send_rdma_nodp(osd_client_t *cl)
 
 void osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
-    if (rdma_context->odp)
+    if (cl->rdma_conn->ctx->odp)
         try_send_rdma_odp(cl);
     else
         try_send_rdma_nodp(cl);
@@ -744,9 +741,9 @@ bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
     {
         msgr_rdma_buf_t b;
         b.buf = malloc_or_die(rc->max_msg);
-        if (!rdma_context->odp)
+        if (!rc->ctx->odp)
         {
-            b.mr = ibv_reg_mr(rdma_context->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
+            b.mr = ibv_reg_mr(rc->ctx->pd, b.buf, rc->max_msg, IBV_ACCESS_LOCAL_WRITE);
             if (!b.mr)
             {
                 fprintf(stderr, "Failed to register RDMA memory region: %s\n", strerror(errno));
@@ -761,7 +758,7 @@ bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
 
 #define RDMA_EVENTS_AT_ONCE 32
 
-void osd_messenger_t::handle_rdma_events()
+void osd_messenger_t::handle_rdma_events(msgr_rdma_context_t *rdma_context)
 {
     // Request next notification
     ibv_cq *ev_cq;

src/client/msgr_rdma.h

@@ -2,9 +2,13 @@
 // License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
 
 #pragma once
+#ifdef WITH_RDMACM
+#include <rdma/rdma_cma.h>
+#endif
 #include <infiniband/verbs.h>
 #include <string>
 #include <vector>
+#include "addr_util.h"
 
 struct msgr_rdma_address_t
 {
@@ -20,7 +24,6 @@ struct msgr_rdma_address_t
 struct msgr_rdma_context_t
 {
     ibv_context *context = NULL;
-    ibv_device *dev = NULL;
     ibv_device_attr_ex attrx;
     ibv_pd *pd = NULL;
     bool odp = false;
@@ -35,8 +38,17 @@ struct msgr_rdma_context_t
     uint32_t mtu;
     int max_cqe = 0;
     int used_max_cqe = 0;
+    addr_mask_t net_mask = {};
+    bool is_cm = false;
+    int cm_refs = 0;
+
+    static std::vector<msgr_rdma_context_t*> create_all(const std::vector<addr_mask_t> & osd_network_masks,
+        const char *sel_dev_name, int sel_port_num, int sel_gid_index, uint32_t sel_mtu, bool odp, int log_level);
+    static msgr_rdma_context_t *create(ibv_device *dev, ibv_port_attr & portinfo,
+        int ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
+    static msgr_rdma_context_t* create_cm(ibv_context *ctx);
+    bool reserve_cqe(int n);
 
-    static msgr_rdma_context_t *create(std::vector<std::string> osd_networks, const char *ib_devname, uint8_t ib_port, int gid_index, uint32_t mtu, bool odp, int log_level);
     ~msgr_rdma_context_t();
 };
 
@@ -50,11 +62,14 @@ struct msgr_rdma_connection_t
 {
     msgr_rdma_context_t *ctx = NULL;
     ibv_qp *qp = NULL;
+#ifdef WITH_RDMACM
+    rdma_cm_id *cmid = NULL;
+#endif
     msgr_rdma_address_t addr;
     int max_send = 0, max_recv = 0, max_sge = 0;
-    int cur_send = 0, cur_recv = 0;
     uint64_t max_msg = 0;
 
+    int cur_send = 0, cur_recv = 0;
     int send_pos = 0, send_buf_pos = 0;
     int next_recv_buf = 0;
     std::vector<msgr_rdma_buf_t> recv_buffers;

src/client/msgr_rdmacm.cpp

@@ -0,0 +1,539 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <fcntl.h>
+#include "msgr_rdma.h"
+#include "messenger.h"
+
+struct rdmacm_connecting_t
+{
+    rdma_cm_id *cmid = NULL;
+    int peer_fd = -1;
+    osd_num_t peer_osd = 0;
+    std::string addr;
+    sockaddr_storage parsed_addr = {};
+    int rdmacm_port = 0;
+    int tcp_port = 0;
+    int timeout_ms = 0;
+    int timeout_id = -1;
+    msgr_rdma_context_t *rdma_context = NULL;
+};
+
+rdma_cm_id *osd_messenger_t::rdmacm_listen(const std::string & bind_address, int rdmacm_port, int *bound_port, int log_level)
+{
+    sockaddr_storage addr = {};
+    rdma_cm_id *listener = NULL;
+    int r = rdma_create_id(rdmacm_evch, &listener, NULL, RDMA_PS_TCP);
+    if (r != 0)
+    {
+        fprintf(stderr, "Failed to create RDMA-CM ID: %s (code %d)\n", strerror(errno), errno);
+        goto fail;
+    }
+    if (!string_to_addr(bind_address, 0, rdmacm_port, &addr))
+    {
+        fprintf(stderr, "Server address: %s is not valid\n", bind_address.c_str());
+        goto fail;
+    }
+    r = rdma_bind_addr(listener, (sockaddr*)&addr);
+    if (r != 0)
+    {
+        fprintf(stderr, "Failed to bind RDMA-CM to %s:%d: %s (code %d)\n", bind_address.c_str(), rdmacm_port, strerror(errno), errno);
+        goto fail;
+    }
+    r = rdma_listen(listener, 128);
+    if (r != 0)
+    {
+        fprintf(stderr, "Failed to listen to RDMA-CM address %s:%d: %s (code %d)\n", bind_address.c_str(), rdmacm_port, strerror(errno), errno);
+        goto fail;
+    }
+    if (bound_port)
+    {
+        *bound_port = ntohs(rdma_get_src_port(listener));
+    }
+    if (log_level > 0)
+    {
+        fprintf(stderr, "Listening to RDMA-CM address %s port %d\n", bind_address.c_str(), *bound_port);
+    }
+    return listener;
+fail:
+    rdma_destroy_id(listener);
+    return NULL;
+}
+
+void osd_messenger_t::rdmacm_destroy_listener(rdma_cm_id *listener)
+{
+    rdma_destroy_id(listener);
+}
+
+void osd_messenger_t::handle_rdmacm_events()
+{
+    // rdma_destroy_id infinitely waits for pthread_cond if called before all events are acked :-(...
+    std::vector<rdma_cm_event> events_copy;
+    while (1)
+    {
+        rdma_cm_event *ev = NULL;
+        int r = rdma_get_cm_event(rdmacm_evch, &ev);
+        if (r != 0)
+        {
+            if (errno == EAGAIN || errno == EINTR)
+                break;
+            fprintf(stderr, "Failed to get RDMA-CM event: %s (code %d)\n", strerror(errno), errno);
+            exit(1);
+        }
+        // ...so we save a copy of all events EXCEPT connection requests, otherwise they sometimes fail with EVENT_DISCONNECT
+        if (ev->event == RDMA_CM_EVENT_CONNECT_REQUEST)
+        {
+            rdmacm_accept(ev);
+        }
+        else
+        {
+            events_copy.push_back(*ev);
+        }
+        r = rdma_ack_cm_event(ev);
+        if (r != 0)
+        {
+            fprintf(stderr, "Failed to ack (free) RDMA-CM event: %s (code %d)\n", strerror(errno), errno);
+            exit(1);
+        }
+    }
+    for (auto & evl: events_copy)
+    {
+        auto ev = &evl;
+        if (ev->event == RDMA_CM_EVENT_CONNECT_REQUEST)
+        {
+            // Do nothing, handled above
+        }
+        else if (ev->event == RDMA_CM_EVENT_CONNECT_ERROR ||
+            ev->event == RDMA_CM_EVENT_REJECTED ||
+            ev->event == RDMA_CM_EVENT_DISCONNECTED ||
+            ev->event == RDMA_CM_EVENT_DEVICE_REMOVAL)
+        {
+            auto event_type_name = ev->event == RDMA_CM_EVENT_CONNECT_ERROR ? "RDMA_CM_EVENT_CONNECT_ERROR" : (
+                ev->event == RDMA_CM_EVENT_REJECTED ? "RDMA_CM_EVENT_REJECTED" : (
+                ev->event == RDMA_CM_EVENT_DISCONNECTED ? "RDMA_CM_EVENT_DISCONNECTED" : "RDMA_CM_EVENT_DEVICE_REMOVAL"));
+            auto cli_it = rdmacm_connections.find(ev->id);
+            if (cli_it != rdmacm_connections.end())
+            {
+                fprintf(stderr, "Received %s event for peer %d, closing connection\n",
+                    event_type_name, cli_it->second->peer_fd);
+                stop_client(cli_it->second->peer_fd);
+            }
+            else if (rdmacm_connecting.find(ev->id) != rdmacm_connecting.end())
+            {
+                fprintf(stderr, "Received %s event for RDMA-CM OSD %ju connection\n",
+                    event_type_name, rdmacm_connecting[ev->id]->peer_osd);
+                rdmacm_established(ev);
+            }
+            else
+            {
+                fprintf(stderr, "Received %s event for an unknown RDMA-CM connection 0x%jx - ignoring\n",
+                    event_type_name, (uint64_t)ev->id);
+            }
+        }
+        else if (ev->event == RDMA_CM_EVENT_ADDR_RESOLVED || ev->event == RDMA_CM_EVENT_ADDR_ERROR)
+        {
+            rdmacm_address_resolved(ev);
+        }
+        else if (ev->event == RDMA_CM_EVENT_ROUTE_RESOLVED || ev->event == RDMA_CM_EVENT_ROUTE_ERROR)
+        {
+            rdmacm_route_resolved(ev);
+        }
+        else if (ev->event == RDMA_CM_EVENT_CONNECT_RESPONSE)
+        {
+            // Just OK
+        }
+        else if (ev->event == RDMA_CM_EVENT_UNREACHABLE || ev->event == RDMA_CM_EVENT_REJECTED)
+        {
+            // Handle error
+            rdmacm_established(ev);
+        }
+        else if (ev->event == RDMA_CM_EVENT_ESTABLISHED)
+        {
+            rdmacm_established(ev);
+        }
+        else if (ev->event == RDMA_CM_EVENT_ADDR_CHANGE || ev->event == RDMA_CM_EVENT_TIMEWAIT_EXIT)
+        {
+            // Do nothing
+        }
+        else
+        {
+            // Other events are unexpected
+            fprintf(stderr, "Unexpected RDMA-CM event type: %d\n", ev->event);
+        }
+    }
+}
+
+msgr_rdma_context_t* msgr_rdma_context_t::create_cm(ibv_context *ctx)
+{
+    auto rdma_context = new msgr_rdma_context_t;
+    rdma_context->is_cm = true;
+    rdma_context->context = ctx;
+    rdma_context->pd = ibv_alloc_pd(ctx);
+    if (!rdma_context->pd)
+    {
+        fprintf(stderr, "Couldn't allocate RDMA protection domain\n");
+        delete rdma_context;
+        return NULL;
+    }
+    rdma_context->odp = false;
+    rdma_context->channel = ibv_create_comp_channel(rdma_context->context);
+    if (!rdma_context->channel)
+    {
+        fprintf(stderr, "Couldn't create RDMA completion channel\n");
+        delete rdma_context;
+        return NULL;
+    }
+    rdma_context->max_cqe = 4096;
+    rdma_context->cq = ibv_create_cq(rdma_context->context, rdma_context->max_cqe, NULL, rdma_context->channel, 0);
+    if (!rdma_context->cq)
+    {
+        fprintf(stderr, "Couldn't create RDMA completion queue\n");
+        delete rdma_context;
+        return NULL;
+    }
+    if (ibv_query_device_ex(rdma_context->context, NULL, &rdma_context->attrx))
+    {
+        fprintf(stderr, "Couldn't query RDMA device for its features\n");
+        delete rdma_context;
+        return NULL;
+    }
+    return rdma_context;
+}
+
+msgr_rdma_context_t* osd_messenger_t::rdmacm_get_context(ibv_context *verbs)
+{
+    // Find the context by device
+    // We assume that RDMA_CM ev->id->verbs is always the same for the same device (but PD for example isn't)
+    msgr_rdma_context_t *rdma_context = NULL;
+    for (auto ctx: rdma_contexts)
+    {
+        if (ctx->context == verbs)
+        {
+            rdma_context = ctx;
+            break;
+        }
+    }
+    if (!rdma_context)
+    {
+        // Wrap into a new msgr_rdma_context_t
+        rdma_context = msgr_rdma_context_t::create_cm(verbs);
+        if (!rdma_context)
+            return NULL;
+        fcntl(rdma_context->channel->fd, F_SETFL, fcntl(rdma_context->channel->fd, F_GETFL, 0) | O_NONBLOCK);
+        tfd->set_fd_handler(rdma_context->channel->fd, false, [this, rdma_context](int notify_fd, int epoll_events)
+        {
+            handle_rdma_events(rdma_context);
+        });
+        handle_rdma_events(rdma_context);
+        rdma_contexts.push_back(rdma_context);
+    }
+    return rdma_context;
+}
+
+msgr_rdma_context_t* osd_messenger_t::rdmacm_create_qp(rdma_cm_id *cmid)
+{
+    auto rdma_context = rdmacm_get_context(cmid->verbs);
+    if (!rdma_context)
+    {
+        return NULL;
+    }
+    rdma_context->reserve_cqe(rdma_max_send+rdma_max_recv);
+    auto max_sge = rdma_max_sge > rdma_context->attrx.orig_attr.max_sge
+        ? rdma_context->attrx.orig_attr.max_sge : rdma_max_sge;
+    ibv_qp_init_attr init_attr = {
+        .send_cq = rdma_context->cq,
+        .recv_cq = rdma_context->cq,
+        .cap     = {
+            .max_send_wr  = (uint32_t)rdma_max_send,
+            .max_recv_wr  = (uint32_t)rdma_max_recv,
+            .max_send_sge = (uint32_t)max_sge,
+            .max_recv_sge = (uint32_t)max_sge,
+        },
+        .qp_type = IBV_QPT_RC,
+    };
+    int r = rdma_create_qp(cmid, rdma_context->pd, &init_attr);
+    if (r != 0)
+    {
+        fprintf(stderr, "Failed to create a queue pair via RDMA-CM: %s (code %d)\n", strerror(errno), errno);
+        rdma_context->reserve_cqe(-rdma_max_send-rdma_max_recv);
+        return NULL;
+    }
+    return rdma_context;
+}
+
+void osd_messenger_t::rdmacm_accept(rdma_cm_event *ev)
+{
+    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
+    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fake_fd < 0)
+    {
+        fprintf(stderr, "Failed to allocate a fake socket for RDMA-CM client: %s (code %d)\n", strerror(errno), errno);
+        rdma_destroy_id(ev->id);
+        return;
+    }
+    auto rdma_context = rdmacm_create_qp(ev->id);
+    if (!rdma_context)
+    {
+        rdma_destroy_id(ev->id);
+        return;
+    }
+    // We don't need private_data, RDMA_READ or ATOMIC so use default 1
+    rdma_conn_param conn_params = {
+        .responder_resources = 1,
+        .initiator_depth = 1,
+        .retry_count = 7,
+        .rnr_retry_count = 7,
+    };
+    if (rdma_accept(ev->id, &conn_params) != 0)
+    {
+        fprintf(stderr, "Failed to accept RDMA-CM connection: %s (code %d)\n", strerror(errno), errno);
+        rdma_context->reserve_cqe(-rdma_max_send-rdma_max_recv);
+        rdma_destroy_qp(ev->id);
+        rdma_destroy_id(ev->id);
+        return;
+    }
+    // Wait for RDMA_CM_ESTABLISHED, and enable the connection only after it
+    auto conn = new rdmacm_connecting_t;
+    conn->cmid = ev->id;
+    conn->peer_fd = fake_fd;
+    conn->parsed_addr = *(sockaddr_storage*)rdma_get_peer_addr(ev->id);
+    conn->rdma_context = rdma_context;
+    rdmacm_set_conn_timeout(conn);
+    rdmacm_connecting[ev->id] = conn;
+    fprintf(stderr, "[OSD %ju] new client %d: connection from %s via RDMA-CM\n", this->osd_num, conn->peer_fd,
+        addr_to_string(conn->parsed_addr).c_str());
+}
+
+void osd_messenger_t::rdmacm_set_conn_timeout(rdmacm_connecting_t *conn)
+{
+    conn->timeout_ms = peer_connect_timeout*1000;
+    if (peer_connect_timeout > 0)
+    {
+        conn->timeout_id = tfd->set_timer(1000*peer_connect_timeout, false, [this, cmid = conn->cmid](int timer_id)
+        {
+            auto conn = rdmacm_connecting.at(cmid);
+            conn->timeout_id = -1;
+            if (conn->peer_osd)
+                fprintf(stderr, "RDMA-CM connection to %s timed out\n", conn->addr.c_str());
+            else
+                fprintf(stderr, "Incoming RDMA-CM connection from %s timed out\n", addr_to_string(conn->parsed_addr).c_str());
+            rdmacm_on_connect_peer_error(cmid, -EPIPE);
+        });
+    }
+}
+
+void osd_messenger_t::rdmacm_on_connect_peer_error(rdma_cm_id *cmid, int res)
+{
+    auto conn = rdmacm_connecting.at(cmid);
+    auto addr = conn->addr;
+    auto tcp_port = conn->tcp_port;
+    auto peer_osd = conn->peer_osd;
+    if (conn->timeout_id >= 0)
+        tfd->clear_timer(conn->timeout_id);
+    if (conn->peer_fd >= 0)
+        close(conn->peer_fd);
+    if (conn->rdma_context)
+        conn->rdma_context->reserve_cqe(-rdma_max_send-rdma_max_recv);
+    if (conn->cmid)
+    {
+        if (conn->cmid->qp)
+            rdma_destroy_qp(conn->cmid);
+        rdma_destroy_id(conn->cmid);
+    }
+    rdmacm_connecting.erase(cmid);
+    delete conn;
+    if (peer_osd)
+    {
+        if (!disable_tcp)
+        {
+            // Fall back to TCP instead of just reporting the error to on_connect_peer()
+            try_connect_peer_tcp(peer_osd, addr.c_str(), tcp_port);
+        }
+        else
+        {
+            // TCP is disabled
+            on_connect_peer(peer_osd, res == 0 ? -EINVAL : (res > 0 ? -res : res));
+        }
+    }
+}
+
+void osd_messenger_t::rdmacm_try_connect_peer(uint64_t peer_osd, const std::string & addr, int rdmacm_port, int fallback_tcp_port)
+{
+    struct sockaddr_storage sa = {};
+    if (!string_to_addr(addr, false, rdmacm_port, &sa))
+    {
+        fprintf(stderr, "Address %s is invalid\n", addr.c_str());
+        on_connect_peer(peer_osd, -EINVAL);
+        return;
+    }
+    rdma_cm_id *cmid = NULL;
+    if (rdma_create_id(rdmacm_evch, &cmid, NULL, RDMA_PS_TCP) != 0)
+    {
+        int res = -errno;
+        fprintf(stderr, "Failed to create RDMA-CM ID: %s (code %d), using TCP\n", strerror(errno), errno);
+        if (!disable_tcp)
+            try_connect_peer_tcp(peer_osd, addr.c_str(), fallback_tcp_port);
+        else
+            on_connect_peer(peer_osd, res);
+        return;
+    }
+    // Make a fake FD (FIXME: do not use FDs for identifying clients!)
+    int fake_fd = socket(AF_INET, SOCK_STREAM, 0);
+    if (fake_fd < 0)
+    {
+        int res = -errno;
+        rdma_destroy_id(cmid);
+        // Can't create socket, pointless to try TCP
+        on_connect_peer(peer_osd, res);
+        return;
+    }
+    if (log_level > 0)
+        fprintf(stderr, "Trying to connect to OSD %ju at %s:%d via RDMA-CM\n", peer_osd, addr.c_str(), rdmacm_port);
+    auto conn = new rdmacm_connecting_t;
+    rdmacm_connecting[cmid] = conn;
+    conn->cmid = cmid;
+    conn->peer_fd = fake_fd;
+    conn->peer_osd = peer_osd;
+    conn->addr = addr;
+    conn->parsed_addr = sa;
+    conn->rdmacm_port = rdmacm_port;
+    conn->tcp_port = fallback_tcp_port;
+    rdmacm_set_conn_timeout(conn);
+    if (rdma_resolve_addr(cmid, NULL, (sockaddr*)&conn->parsed_addr, conn->timeout_ms) != 0)
+    {
+        auto res = -errno;
+        // ENODEV means that the client doesn't have an RDMA device for this address
+        if (res != -ENODEV || log_level > 0)
+            fprintf(stderr, "Failed to resolve address %s via RDMA-CM: %s (code %d)\n", addr.c_str(), strerror(errno), errno);
+        rdmacm_on_connect_peer_error(cmid, res);
+        return;
+    }
+}
+
+void osd_messenger_t::rdmacm_address_resolved(rdma_cm_event *ev)
+{
+    auto cmid = ev->id;
+    auto conn_it = rdmacm_connecting.find(cmid);
+    if (conn_it == rdmacm_connecting.end())
+    {
+        // Silently ignore unknown IDs
+        return;
+    }
+    auto conn = conn_it->second;
+    if (ev->event != RDMA_CM_EVENT_ADDR_RESOLVED || ev->status != 0)
+    {
+        fprintf(stderr, "Failed to resolve address %s via RDMA-CM: %s (code %d)\n", conn->addr.c_str(),
+            ev->status > 0 ? "unknown error" : strerror(-ev->status), ev->status);
+        rdmacm_on_connect_peer_error(cmid, ev->status);
+        return;
+    }
+    auto rdma_context = rdmacm_create_qp(cmid);
+    if (!rdma_context)
+    {
+        rdmacm_on_connect_peer_error(cmid, -EIO);
+        return;
+    }
+    conn->rdma_context = rdma_context;
+    if (rdma_resolve_route(cmid, conn->timeout_ms) != 0)
+    {
+        int res = -errno;
+        fprintf(stderr, "Failed to resolve route to %s via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), strerror(errno), errno);
+        rdmacm_on_connect_peer_error(cmid, res);
+        return;
+    }
+}
+
+void osd_messenger_t::rdmacm_route_resolved(rdma_cm_event *ev)
+{
+    auto cmid = ev->id;
+    auto conn_it = rdmacm_connecting.find(cmid);
+    if (conn_it == rdmacm_connecting.end())
+    {
+        // Silently ignore unknown IDs
+        return;
+    }
+    auto conn = conn_it->second;
+    if (ev->event != RDMA_CM_EVENT_ROUTE_RESOLVED || ev->status != 0)
+    {
+        fprintf(stderr, "Failed to resolve route to %s via RDMA-CM: %s (code %d)\n", conn->addr.c_str(),
+            ev->status > 0 ? "unknown error" : strerror(-ev->status), ev->status);
+        rdmacm_on_connect_peer_error(cmid, ev->status);
+        return;
+    }
+    // We don't need private_data, RDMA_READ or ATOMIC so use default 1
+    rdma_conn_param conn_params = {
+        .responder_resources = 1,
+        .initiator_depth = 1,
+        .retry_count = 7,
+        .rnr_retry_count = 7,
+    };
+    if (rdma_connect(cmid, &conn_params) != 0)
+    {
+        int res = -errno;
+        fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->rdmacm_port, strerror(errno), errno);
+        rdmacm_on_connect_peer_error(cmid, res);
+        return;
+    }
+}
+
+void osd_messenger_t::rdmacm_established(rdma_cm_event *ev)
+{
+    auto cmid = ev->id;
+    auto conn_it = rdmacm_connecting.find(cmid);
+    if (conn_it == rdmacm_connecting.end())
+    {
+        // Silently ignore unknown IDs
+        return;
+    }
+    auto conn = conn_it->second;
+    auto peer_osd = conn->peer_osd;
+    if (ev->event != RDMA_CM_EVENT_ESTABLISHED || ev->status != 0)
+    {
+        fprintf(stderr, "Failed to connect to %s:%d via RDMA-CM: %s (code %d)\n", conn->addr.c_str(), conn->rdmacm_port,
+            ev->status > 0 ? "unknown error" : strerror(-ev->status), ev->status);
+        rdmacm_on_connect_peer_error(cmid, ev->status);
+        return;
+    }
+    // Wrap into a new msgr_rdma_connection_t
+    msgr_rdma_connection_t *rc = new msgr_rdma_connection_t;
+    rc->ctx = conn->rdma_context;
+    rc->ctx->cm_refs++; // FIXME now unused, count also connecting_t's when used
+    rc->max_send = rdma_max_send;
+    rc->max_recv = rdma_max_recv;
+    rc->max_sge = rdma_max_sge > rc->ctx->attrx.orig_attr.max_sge
+        ? rc->ctx->attrx.orig_attr.max_sge : rdma_max_sge;
+    rc->max_msg = rdma_max_msg;
+    rc->cmid = conn->cmid;
+    rc->qp = conn->cmid->qp;
+    // And an osd_client_t
+    auto cl = new osd_client_t();
+    cl->peer_addr = conn->parsed_addr;
+    cl->peer_port = conn->rdmacm_port;
+    cl->peer_fd = conn->peer_fd;
+    cl->peer_state = PEER_RDMA;
+    cl->connect_timeout_id = -1;
+    cl->osd_num = peer_osd;
+    cl->in_buf = malloc_or_die(receive_buffer_size);
+    cl->rdma_conn = rc;
+    clients[conn->peer_fd] = cl;
+    if (conn->timeout_id >= 0)
+    {
+        tfd->clear_timer(conn->timeout_id);
+    }
+    delete conn;
+    rdmacm_connecting.erase(cmid);
+    rdmacm_connections[cmid] = cl;
+    if (log_level > 0 && peer_osd)
+    {
+        fprintf(stderr, "Successfully connected with OSD %ju using RDMA-CM\n", peer_osd);
+    }
+    // Add initial receive request(s)
+    try_recv_rdma(cl);
+    if (peer_osd)
+    {
+        check_peer_config(cl);
+    }
+}

src/client/msgr_send.cpp

@@ -187,6 +187,7 @@ bool osd_messenger_t::try_send(osd_client_t *cl)
     {
         return true;
     }
+    assert(cl->peer_state != PEER_RDMA);
     if (ringloop && !use_sync_send_recv)
     {
         auto iothread = iothreads.size() ? iothreads[peer_fd % iothreads.size()] : NULL;

src/client/qemu_driver.c

@@ -568,6 +568,22 @@ static int vitastor_file_open(BlockDriverState *bs, QDict *options, int flags, E
 static void vitastor_close(BlockDriverState *bs)
 {
     VitastorClient *client = bs->opaque;
+    if (client->uring_eventfd >= 0)
+    {
+        // clear the eventfd handler
+        universal_aio_set_fd_handler(client->ctx, client->uring_eventfd, NULL, NULL, NULL);
+        int wait_bh = 0;
+        qemu_mutex_lock(&client->mutex);
+        // clear uring_eventfd itself to prevent future scheduling of new B/H
+        client->uring_eventfd = -1;
+        wait_bh = client->bh_uring_scheduled;
+        qemu_mutex_unlock(&client->mutex);
+        if (wait_bh)
+        {
+            // wait until existing scheduled B/H is ran
+            BDRV_POLL_WHILE(bs, client->bh_uring_scheduled);
+        }
+    }
     vitastor_c_destroy(client->proxy);
     if (client->fds)
     {

src/client/vitastor.pc.in

@@ -6,7 +6,7 @@ includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
 
 Name: Vitastor
 Description: Vitastor client library
-Version: 2.0.0
+Version: 2.1.0
 Libs: -L${libdir} -lvitastor_client
 Cflags: -I${includedir}
 

src/cmd/cli.h

@@ -98,5 +98,3 @@ std::string format_lat(uint64_t lat);
 std::string format_q(double depth);
 
 bool stupid_glob(const std::string str, const std::string glob);
-
-std::string implode(const std::string & sep, json11::Json array);

src/cmd/cli_modify_osd.cpp

@@ -58,6 +58,12 @@ struct osd_changer_t
             state = 100;
             return;
         }
+        if (set_reweight && new_reweight > 1)
+        {
+            result = (cli_result_t){ .err = EINVAL, .text = "Reweight can't be larger than 1" };
+            state = 100;
+            return;
+        }
         parent->etcd_txn(json11::Json::object {
             { "success", json11::Json::array {
                 json11::Json::object {

src/cmd/cli_osd_tree.cpp

@@ -7,6 +7,7 @@
 #include "epoll_manager.h"
 #include "pg_states.h"
 #include "str_util.h"
+#include "json_util.h"
 
 struct placement_osd_t
 {

src/cmd/cli_pg_ls.cpp

@@ -5,6 +5,7 @@
 #include "cluster_client.h"
 #include "pg_states.h"
 #include "str_util.h"
+#include "json_util.h"
 
 struct pg_lister_t
 {
@@ -104,7 +105,7 @@ resume_1:
                                 {
                                     if (pg_state_names[i] == bit)
                                     {
-                                        mask |= (uint64_t)1 << i;
+                                        mask |= pg_state_bits[i];
                                         found = true;
                                         break;
                                     }
@@ -173,7 +174,7 @@ resume_1:
                     json11::Json::array state_names;
                     for (int i = 0; i < pg_state_bit_count; i++)
                     {
-                        if (pgp.second.cur_state & (1 << i))
+                        if (pgp.second.cur_state & pg_state_bits[i])
                         {
                             state_names.push_back(std::string(pg_state_names[i]));
                         }

src/cmd/cli_pool_create.cpp

@@ -10,6 +10,7 @@
 #include "epoll_manager.h"
 #include "pg_states.h"
 #include "str_util.h"
+#include "json_util.h"
 
 struct pool_creator_t
 {

src/cmd/cli_pool_ls.cpp

@@ -5,6 +5,7 @@
 #include "cli.h"
 #include "cluster_client.h"
 #include "str_util.h"
+#include "json_util.h"
 #include "pg_states.h"
 
 // List pools with space statistics
@@ -665,19 +666,3 @@ std::function<bool(cli_result_t &)> cli_tool_t::start_pool_ls(json11::Json cfg)
         return false;
     };
 }
-
-std::string implode(const std::string & sep, json11::Json array)
-{
-    if (array.is_number() || array.is_bool() || array.is_string())
-    {
-        return array.as_string();
-    }
-    std::string res;
-    bool first = true;
-    for (auto & item: array.array_items())
-    {
-        res += (first ? item.as_string() : sep+item.as_string());
-        first = false;
-    }
-    return res;
-}

src/cmd/cli_rm_wildcard.cpp

@@ -138,7 +138,7 @@ struct wildcard_remover_t
                     if (inode_it != parent->cli->st_cli.inode_config.end())
                         fprintf(stderr, "Warning: image %s modified by someone else during deletion, restarting wildcard deletion\n", inode_it->second.name.c_str());
                     else
-                        fprintf(stderr, "Warning: inode %lx modified by someone else during deletion, retrying wildcard deletion\n", irev.inode_num);
+                        fprintf(stderr, "Warning: inode %jx modified by someone else during deletion, retrying wildcard deletion\n", irev.inode_num);
                     goto resume_0;
                 }
             }

src/nfs/nfs_fsstat.cpp

@@ -22,8 +22,8 @@ int nfs3_fsstat_proc(void *opaque, rpc_op_t *rop)
     {
         auto ttb = pst_it->second["total_raw_tb"].number_value();
         auto ftb = (pst_it->second["total_raw_tb"].number_value() - pst_it->second["used_raw_tb"].number_value());
-        tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
-        fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)2<<40);
+        tbytes = ttb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)1<<40);
+        fbytes = ftb / pst_it->second["raw_to_usable"].number_value() * ((uint64_t)1<<40);
     }
     *reply = (FSSTAT3res){
         .status = NFS3_OK,

src/nfs/nfs_proxy_rdma.cpp

@@ -47,7 +47,7 @@ struct nfs_rdma_dev_state_t
     ibv_comp_channel *channel = NULL;
     ibv_cq *cq = NULL;
     rdma_allocator_t *alloc = NULL;
-    int max_cqe = 0, used_max_cqe = 0;
+    int max_cqe = 0;
 
     static nfs_rdma_dev_state_t *create(rdma_cm_id *cmid, uint64_t rdma_malloc_round_to, uint64_t rdma_max_unused_buffers);
     ~nfs_rdma_dev_state_t();

src/osd/CMakeLists.txt

@@ -14,6 +14,7 @@ target_link_libraries(vitastor-osd
 	Jerasure
 	${ISAL_LIBRARIES}
 	${IBVERBS_LIBRARIES}
+	${RDMACM_LIBRARIES}
 )
 
 # osd_rmw_test

src/osd/osd.cpp

@@ -120,7 +120,14 @@ osd_t::~osd_t()
     delete epmgr;
     if (bs)
         delete bs;
-    close(listen_fd);
+#ifdef WITH_RDMACM
+    for (rdma_cm_id *listener: rdmacm_listeners)
+        msgr.rdmacm_destroy_listener(listener);
+    rdmacm_listeners.clear();
+#endif
+    for (auto listen_fd: listen_fds)
+        close(listen_fd);
+    listen_fds.clear();
     free(zero_buffer);
 }
 
@@ -162,12 +169,23 @@ void osd_t::parse_config(bool init)
         else
             immediate_commit = IMMEDIATE_NONE;
         // Bind address
-        bind_address = config["bind_address"].string_value();
-        if (bind_address == "")
-            bind_address = "0.0.0.0";
+        cfg_bind_addresses.clear();
+        if (config.find("bind_address") != config.end())
+        {
+            if (config["bind_address"].is_string())
+                cfg_bind_addresses.push_back(config["bind_address"].string_value());
+            else if (config["bind_address"].is_array())
+                for (auto & addr: config["bind_address"].array_items())
+                    cfg_bind_addresses.push_back(addr.string_value());
+        }
         bind_port = config["bind_port"].uint64_value();
         if (bind_port <= 0 || bind_port > 65535)
             bind_port = 0;
+#ifdef WITH_RDMACM
+        // Use RDMA CM? (required for iWARP and may be useful for IB)
+        this->use_rdmacm = config["use_rdmacm"].bool_value() || config["use_rdmacm"].uint64_value() != 0;
+        this->disable_tcp = this->use_rdmacm && (config["disable_tcp"].bool_value() || config["disable_tcp"].uint64_value() != 0);
+#endif
         // OSD configuration
         etcd_report_interval = config["etcd_report_interval"].uint64_value();
         if (etcd_report_interval <= 0)
@@ -186,6 +204,7 @@ void osd_t::parse_config(bool init)
     no_recovery = json_is_true(config["no_recovery"]);
     auto old_no_scrub = no_scrub;
     no_scrub = json_is_true(config["no_scrub"]);
+    allow_net_split = json_is_true(config["allow_net_split"]);
     auto old_autosync_interval = autosync_interval;
     if (!config["autosync_interval"].is_null())
     {
@@ -321,41 +340,54 @@ void osd_t::parse_config(bool init)
 
 void osd_t::bind_socket()
 {
-    if (config["osd_network"].is_string() ||
-        config["osd_network"].is_array())
+    if (cfg_bind_addresses.size())
     {
-        std::vector<std::string> mask;
-        if (config["osd_network"].is_string())
-            mask.push_back(config["osd_network"].string_value());
-        else
-            for (auto v: config["osd_network"].array_items())
-                mask.push_back(v.string_value());
-        auto matched_addrs = getifaddr_list(mask);
-        if (matched_addrs.size() > 1)
+        bind_addresses = cfg_bind_addresses;
+    }
+    else if (msgr.all_osd_network_masks.size())
+    {
+        bind_addresses = getifaddr_list(msgr.all_osd_network_masks);
+        if (!bind_addresses.size())
         {
-            fprintf(stderr, "More than 1 address matches requested network(s): %s\n", json11::Json(matched_addrs).dump().c_str());
-            force_stop(1);
-        }
-        if (!matched_addrs.size())
-        {
-            std::string nets;
-            for (auto v: mask)
-                nets += (nets == "" ? v : ","+v);
+            auto nets = implode(", ", msgr.all_osd_networks);
             fprintf(stderr, "Addresses matching osd_network(s) %s not found\n", nets.c_str());
             force_stop(1);
         }
-        bind_address = matched_addrs[0];
     }
-
-    // FIXME Support multiple listening sockets
-
-    listen_fd = create_and_bind_socket(bind_address, bind_port, listen_backlog, &listening_port);
-    fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
-
-    epmgr->set_fd_handler(listen_fd, false, [this](int fd, int events)
+    else
     {
-        msgr.accept_connections(listen_fd);
-    });
+        bind_addresses.push_back("0.0.0.0");
+    }
+    if (!disable_tcp)
+    {
+        for (auto & bind_address: bind_addresses)
+        {
+            int listen_fd = create_and_bind_socket(bind_address, listening_port ? listening_port : bind_port, listen_backlog, &listening_port);
+            fcntl(listen_fd, F_SETFL, fcntl(listen_fd, F_GETFL, 0) | O_NONBLOCK);
+            epmgr->set_fd_handler(listen_fd, false, [this](int fd, int events)
+            {
+                msgr.accept_connections(fd);
+            });
+            listen_fds.push_back(listen_fd);
+        }
+    }
+#ifdef WITH_RDMACM
+    if (use_rdmacm)
+    {
+        for (auto & bind_address: bind_addresses)
+        {
+            // iWARP shares port numbers with TCP traffic and can't listen on the same port with TCP and RDMACM
+            auto listener = msgr.rdmacm_listen(bind_address, rdmacm_port, &rdmacm_port, log_level);
+            if (listener)
+                rdmacm_listeners.push_back(listener);
+        }
+        if (!rdmacm_listeners.size() && disable_tcp)
+        {
+            fprintf(stderr, "Failed to create RDMA-CM listeners, exiting\n");
+            force_stop(1);
+        }
+    }
+#endif
 }
 
 bool osd_t::shutdown()

src/osd/osd.h

@@ -106,8 +106,11 @@ class osd_t
     bool no_rebalance = false;
     bool no_recovery = false;
     bool no_scrub = false;
-    std::string bind_address;
+    bool allow_net_split = false;
+    std::vector<std::string> cfg_bind_addresses;
     int bind_port, listen_backlog = 128;
+    bool use_rdmacm = false;
+    bool disable_tcp = false;
     // FIXME: Implement client queue depth limit
     int client_queue_depth = 128;
     bool allow_test_ops = false;
@@ -199,7 +202,12 @@ class osd_t
     epoll_manager_t *epmgr = NULL;
 
     int listening_port = 0;
-    int listen_fd = 0;
+    std::vector<std::string> bind_addresses;
+    std::vector<int> listen_fds;
+#ifdef WITH_RDMACM
+    std::vector<rdma_cm_id *> rdmacm_listeners;
+    int rdmacm_port = 0;
+#endif
     ring_consumer_t consumer;
 
     // op statistics

src/osd/osd_cluster.cpp

@@ -165,13 +165,17 @@ json11::Json osd_t::get_osd_state()
     hostname.resize(strnlen(hostname.data(), hostname.size()));
     json11::Json::object st;
     st["state"] = "up";
-    if (bind_address != "0.0.0.0")
-        st["addresses"] = json11::Json::array { bind_address };
+    if (bind_addresses.size() != 1 || bind_addresses[0] != "0.0.0.0")
+        st["addresses"] = bind_addresses;
     else
         st["addresses"] = getifaddr_list();
     st["host"] = std::string(hostname.data(), hostname.size());
     st["version"] = VITASTOR_VERSION;
     st["port"] = listening_port;
+#ifdef WITH_RDMACM
+    if (rdmacm_listeners.size())
+        st["rdmacm_port"] = rdmacm_port;
+#endif
     st["primary_enabled"] = run_primary;
     st["blockstore_enabled"] = bs ? true : false;
     return st;

src/osd/osd_main.cpp

@@ -58,7 +58,7 @@ int main(int narg, char *args[])
         return 1;
     }
     char osdname[16] = { 0 };
-    snprintf(osdname, 16, "osd%lu", config["osd_num"].uint64_value());
+    snprintf(osdname, 16, "osd%ju", config["osd_num"].uint64_value());
     prctl(PR_SET_NAME, (unsigned long)osdname, 0, 0, 0);
     signal(SIGINT, handle_sigint);
     signal(SIGTERM, handle_sigint);

src/osd/osd_peering.cpp

@@ -199,6 +199,7 @@ void osd_t::start_pg_peering(pg_t & pg)
     drop_dirty_pg_connections({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
     // Try to connect with current peers if they're up, but we don't have connections to them
     // Otherwise we may erroneously decide that the pg is incomplete :-)
+    bool all_connected = true;
     for (auto pg_osd: pg.all_peers)
     {
         if (pg_osd != this->osd_num &&
@@ -206,8 +207,17 @@ void osd_t::start_pg_peering(pg_t & pg)
             msgr.wanted_peers.find(pg_osd) == msgr.wanted_peers.end())
         {
             msgr.connect_peer(pg_osd, st_cli.peer_states[pg_osd]);
+            if (!st_cli.peer_states[pg_osd].is_null())
+                all_connected = false;
         }
     }
+    if (!all_connected && !allow_net_split)
+    {
+        // Wait until all OSDs are either connected or their /osd/state disappears from etcd
+        pg.state = PG_INCOMPLETE;
+        report_pg_state(pg);
+        return;
+    }
     // Calculate current write OSD set
     pg.pg_cursize = 0;
     pg.cur_set.resize(pg.target_set.size());

src/test/CMakeLists.txt

@@ -25,6 +25,7 @@ target_link_libraries(stub_uring_osd
 	vitastor_common
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
+	${RDMACM_LIBRARIES}
 	tcmalloc_minimal
 )
 

src/test/test_cluster_client.cpp

@@ -256,7 +256,7 @@ void test1()
 
     pretend_disconnected(cli, 1);
     pretend_connected(cli, 1);
-    cli->continue_ops(true);
+    cli->continue_ops(cli->client_retry_interval);
 
     // Check replay
     {
@@ -319,6 +319,7 @@ void test1()
     pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), -EPIPE);
     check_disconnected(cli, 1);
     pretend_connected(cli, 1);
+    cli->continue_ops(cli->client_retry_interval);
     check_op_count(cli, 1, 1);
     can_complete(r1);
     pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), 0);
@@ -337,7 +338,7 @@ void test1()
     check_completed(r1);
     check_disconnected(cli, 1);
     pretend_connected(cli, 1);
-    cli->continue_ops(true);
+    cli->continue_ops(cli->client_retry_interval);
     check_op_count(cli, 1, 1);
     pretend_op_completed(cli, find_op(cli, 1, OSD_OP_WRITE, 0, 0x1000), 0);
     check_op_count(cli, 1, 1);

src/util/addr_util.cpp

@@ -93,6 +93,13 @@ bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits)
     return true;
 }
 
+bool cidr_sockaddr_match(const sockaddr_storage &addr, const addr_mask_t &mask)
+{
+    return mask.family == addr.ss_family && (mask.family == AF_INET
+        ? cidr_match(((sockaddr_in*)&addr)->sin_addr, mask.ipv4, mask.bits)
+        : cidr6_match(((sockaddr_in6*)&addr)->sin6_addr, mask.ipv6, mask.bits));
+}
+
 addr_mask_t cidr_parse(std::string mask)
 {
     unsigned bits = 255;
@@ -126,13 +133,11 @@ addr_mask_t cidr_parse(std::string mask)
     }
 }
 
-std::vector<std::string> getifaddr_list(std::vector<std::string> mask_cfg, bool include_v6)
+std::vector<std::string> getifaddr_list(const std::vector<addr_mask_t> & masks, bool include_v6)
 {
-    std::vector<addr_mask_t> masks;
-    for (auto mask: mask_cfg)
+    for (auto & mask: masks)
     {
-        masks.push_back(cidr_parse(mask));
-        if (masks[masks.size()-1].family == AF_INET6)
+        if (mask.family == AF_INET6)
         {
             // Auto-enable IPv6 addresses
             include_v6 = true;

src/util/addr_util.h

@@ -18,5 +18,6 @@ std::string addr_to_string(const sockaddr_storage &addr);
 addr_mask_t cidr_parse(std::string mask);
 bool cidr_match(const in_addr &address, const in_addr &network, uint8_t bits);
 bool cidr6_match(const in6_addr &address, const in6_addr &network, uint8_t bits);
-std::vector<std::string> getifaddr_list(std::vector<std::string> mask_cfg = std::vector<std::string>(), bool include_v6 = false);
+bool cidr_sockaddr_match(const sockaddr_storage &addr, const addr_mask_t &mask);
+std::vector<std::string> getifaddr_list(const std::vector<addr_mask_t> & masks = std::vector<addr_mask_t>(), bool include_v6 = false);
 int create_and_bind_socket(std::string bind_address, int bind_port, int listen_backlog, int *listening_port);

src/util/json_util.cpp

@@ -33,3 +33,19 @@ bool json_is_false(const json11::Json & val)
         return !val.bool_value();
     return false;
 }
+
+std::string implode(const std::string & sep, json11::Json array)
+{
+    if (array.is_number() || array.is_bool() || array.is_string())
+    {
+        return array.as_string();
+    }
+    std::string res;
+    bool first = true;
+    for (auto & item: array.array_items())
+    {
+        res += (first ? item.as_string() : sep+item.as_string());
+        first = false;
+    }
+    return res;
+}

src/util/json_util.h

@@ -11,3 +11,4 @@
 std::map<std::string, std::string> json_to_string_map(const json11::Json::object & config);
 bool json_is_true(const json11::Json & val);
 bool json_is_false(const json11::Json & val);
+std::string implode(const std::string & sep, json11::Json array);