Add debug

.gitea/workflows/buildenv.Dockerfile

@@ -1,36 +0,0 @@
-FROM node:16-bullseye
-
-WORKDIR /root
-
-ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
-
-RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
-    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
-    echo >> /etc/apt/preferences; \
-    echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
-    echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
-    echo >> /etc/apt/preferences; \
-    echo 'Package: *' >> /etc/apt/preferences; \
-    echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
-    echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
-    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
-    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
-    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
-
-RUN apt-get update
-RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
-    liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
-RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get -y install jq lp-solve sudo
-RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-
-RUN set -ex; \
-    mkdir qemu-build; \
-    cd qemu-build; \
-    dpkg-source -x /root/qemu*.dsc; \
-    cd qemu*/; \
-    debian/rules configure-qemu || debian/rules b/configure-stamp; \
-    cd b/qemu; \
-    make -j8 config-poison.h || true; \
-    make -j8 qapi/qapi-builtin-types.h

.gitea/workflows/test.Dockerfile

@@ -1,16 +0,0 @@
-FROM git.yourcmc.ru/vitalif/vitastor/buildenv
-
-ADD . /root/vitastor
-
-RUN set -e -x; \
-    mkdir -p /root/fio-build/; \
-    cd /root/fio-build/; \
-    dpkg-source -x /root/fio*.dsc; \
-    cd /root/vitastor; \
-    ln -s /root/fio-build/fio-*/ ./fio; \
-    ln -s /root/qemu-build/qemu-*/ ./qemu; \
-    ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
-    mkdir build; \
-    cd build; \
-    cmake .. -DWITH_ASAN=yes -DWITH_QEMU=yes; \
-    make -j16

.gitea/workflows/test.yml

@@ -1,552 +0,0 @@
-name: Test
-
-on:
-  push:
-    branches:
-    - '*'
-    paths:
-    - '.gitea/**'
-    - 'src/**'
-    - 'mon/**'
-    - 'json11'
-    - 'cpp-btree'
-    - 'tests/**'
-
-env:
-  BUILDENV_IMAGE: git.yourcmc.ru/vitalif/vitastor/buildenv
-  TEST_IMAGE: git.yourcmc.ru/vitalif/vitastor/test
-  OSD_ARGS: '--etcd_quick_timeout 2000'
-
-concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-
-  buildenv:
-    runs-on: ubuntu-latest
-    container: git.yourcmc.ru/vitalif/gitea-ci-dind
-    steps:
-    - uses: actions/checkout@v3
-
-    - name: Build and push
-      run: |
-        set -ex
-        if ! docker manifest inspect $BUILDENV_IMAGE >/dev/null; then
-          docker build -t $BUILDENV_IMAGE -f .gitea/workflows/buildenv.Dockerfile .
-          docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}"
-          docker push $BUILDENV_IMAGE
-        fi
-
-  build:
-    runs-on: ubuntu-latest
-    needs: buildenv
-    container: git.yourcmc.ru/vitalif/gitea-ci-dind
-    steps:
-    - uses: actions/checkout@v3
-      with:
-        submodules: true
-
-    - name: Build and push
-      run: |
-        set -ex
-        if ! docker manifest inspect $TEST_IMAGE:$GITHUB_SHA >/dev/null; then
-          docker build -t $TEST_IMAGE:$GITHUB_SHA -f .gitea/workflows/test.Dockerfile .
-          docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}"
-          docker push $TEST_IMAGE:$GITHUB_SHA
-        fi
-
-  make_test:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    # leak sanitizer sometimes crashes
-    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
-
-  test_add_osd:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_add_osd.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_cas:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_cas.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_change_pg_count:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_change_pg_count.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_change_pg_count_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_change_pg_count.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_change_pg_size:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_change_pg_size.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_create_nomaxid:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_create_nomaxid.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_etcd_fail:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: /root/vitastor/tests/test_etcd_fail.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_failure_domain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_failure_domain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_interrupted_rebalance:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_interrupted_rebalance_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_interrupted_rebalance_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_interrupted_rebalance_ec_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_minsize_1:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_minsize_1.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_move_reappear:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_move_reappear.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rebalance_verify_ec_imm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_rm:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_rm.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_snapshot_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=ec /root/vitastor/tests/test_snapshot.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_splitbrain:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_splitbrain.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_write:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_write.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_write_xor:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: SCHEME=xor /root/vitastor/tests/test_write.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_write_no_same:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 3
-      run: /root/vitastor/tests/test_write_no_same.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_pg_size_2:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: PG_SIZE=2 /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-
-  test_heal_ec:
-    runs-on: ubuntu-latest
-    needs: build
-    container: ${{env.TEST_IMAGE}}:${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: 10
-      run: SCHEME=ec /root/vitastor/tests/test_heal.sh
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- $i --------"
-          cat $i
-          echo ""
-        done
-

.gitea/workflows/tests-to-yaml.pl

@@ -1,68 +0,0 @@
-#!/usr/bin/perl
-
-use strict;
-
-for my $line (<>)
-{
-    if ($line =~ /\.\/(test_[^\.]+)/s)
-    {
-        chomp $line;
-        my $test_name = $1;
-        my $timeout = 3;
-        if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_interrupted_rebalance')
-        {
-            $timeout = 10;
-        }
-        while ($line =~ /([^\s=]+)=(\S+)/gs)
-        {
-            if ($1 eq 'SCHEME' && $2 eq 'ec')
-            {
-                $test_name .= '_ec';
-            }
-            elsif ($1 eq 'SCHEME' && $2 eq 'xor')
-            {
-                $test_name .= '_xor';
-            }
-            elsif ($1 eq 'IMMEDIATE_COMMIT')
-            {
-                $test_name .= '_imm';
-            }
-            else
-            {
-                $test_name .= '_'.lc($1).'_'.$2;
-            }
-        }
-        $line =~ s!\./test_!/root/vitastor/tests/test_!;
-        # Gitea CI doesn't support artifacts yet, lol
-        #- name: Upload results
-        #  uses: actions/upload-artifact\@v3
-        #  if: always()
-        #  with:
-        #    name: ${test_name}_result
-        #    path: |
-        #      /root/vitastor/testdata
-        #      !/root/vitastor/testdata/*.bin
-        #    retention-days: 5
-        print <<"EOF"
-  $test_name:
-    runs-on: ubuntu-latest
-    needs: build
-    container: \${{env.TEST_IMAGE}}:\${{github.sha}}
-    steps:
-    - name: Run test
-      id: test
-      timeout-minutes: $timeout
-      run: $line
-    - name: Print logs
-      if: always() && steps.test.outcome == 'failure'
-      run: |
-        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
-          echo "-------- \$i --------"
-          cat \$i
-          echo ""
-        done
-
-EOF
-;
-    }
-}

CMakeLists.txt

@@ -1,7 +1,7 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
-set(VERSION "0.8.9")
+set(VERSION "0.8.3")
 
 add_subdirectory(src)

VNPL-1.1-RU.txt

@@ -48,9 +48,9 @@ Vitastor, составлены для того, чтобы убедиться,
 интерфейс (прокси), опять же, без открытия в свободный публичный доступ как
 самой программы, так и прокси.
 
-  Сетевая Публичная Лицензия Vitastor разработана специально, чтобы
+  Сетевая Публичная Лицензия Vitastor разработана специально чтобы
 гарантировать, что в таких случаях и модифицированная версия программы, и
-прокси останутся доступными сообществу. Для этого лицензия требует от
+прокси оставались доступными сообществу. Для этого лицензия требует от
 операторов сетевых серверов предоставлять исходный код оригинальной программы,
 а также всех других программ, взаимодействующих с ней на их серверах,
 пользователям этих серверов, на условиях свободных лицензий. Таким образом,

csi/Makefile

@@ -1,4 +1,4 @@
-VERSION ?= v0.8.9
+VERSION ?= v0.8.3
 
 all: build push
 

csi/deploy/004-csi-nodeplugin.yaml

@@ -49,7 +49,7 @@ spec:
             capabilities:
               add: ["SYS_ADMIN"]
             allowPrivilegeEscalation: true
-          image: vitalif/vitastor-csi:v0.8.9
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/deploy/007-csi-provisioner.yaml

@@ -116,7 +116,7 @@ spec:
             privileged: true
             capabilities:
               add: ["SYS_ADMIN"]
-          image: vitalif/vitastor-csi:v0.8.9
+          image: vitalif/vitastor-csi:v0.8.3
           args:
             - "--node=$(NODE_ID)"
             - "--endpoint=$(CSI_ENDPOINT)"

csi/go.mod

@@ -4,10 +4,25 @@ go 1.15
 
 require (
 	github.com/container-storage-interface/spec v1.4.0
+	github.com/coreos/bbolt v0.0.0-00010101000000-000000000000 // indirect
+	github.com/coreos/etcd v3.3.25+incompatible // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
+	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
+	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/kubernetes-csi/csi-lib-utils v0.9.1
+	github.com/soheilhy/cmux v0.1.5 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/bbolt v0.0.0-00010101000000-000000000000 // indirect
+	go.etcd.io/etcd v3.3.25+incompatible
 	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
-	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/grpc v1.33.1
 	k8s.io/klog v1.0.0
 	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10

csi/go.sum

@@ -31,11 +31,14 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
 github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
@@ -43,12 +46,25 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
 github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
 github.com/container-storage-interface/spec v1.4.0 h1:ozAshSKxpJnYUfmkpZCTYyF/4MYeYlhdXbAvPvfGmkg=
 github.com/container-storage-interface/spec v1.4.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/coreos/bbolt v1.3.5 h1:XFv7xaq7701j8ZSEzR28VohFYSlyakMyqNMU5FQH6Ac=
+github.com/coreos/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+github.com/coreos/etcd v3.3.25+incompatible h1:0GQEw6h3YnuOVdtwygkIfJ+Omx0tZ8/QkVyXI4LkbeY=
+github.com/coreos/etcd v3.3.25+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU=
+github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
 github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
+github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
@@ -57,6 +73,7 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -71,10 +88,14 @@ github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nA
 github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls=
 github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7 h1:5ZkaAPbicIKTF2I64qf5Fh8Aa83Q/dnOafMYV0OMwjA=
 github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
@@ -92,6 +113,7 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -105,24 +127,38 @@ github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OI
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw=
+github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
@@ -135,11 +171,14 @@ github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXv
 github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
 github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI=
 github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
 github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
@@ -149,28 +188,38 @@ github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+W
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA=
 github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
 github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I=
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js=
+github.com/soheilhy/cmux v0.1.5/go.mod h1:T7TcVDs9LWfQgPlPsdngu6I6QIoyIFZDDC6sNE1GqG0=
 github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
@@ -182,11 +231,24 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY=
+go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
 go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
@@ -194,6 +256,7 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -213,6 +276,8 @@ golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCc
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -226,20 +291,26 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
 golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -255,9 +326,11 @@ golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8=
 golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -268,6 +341,7 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -286,10 +360,14 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -310,6 +388,8 @@ google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
 google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
 google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
@@ -335,6 +415,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
@@ -363,4 +444,5 @@ k8s.io/utils v0.0.0-20210305010621-2afb4311ab10/go.mod h1:jPW/WVKK9YHAvNhRxK0md/
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

csi/src/config.go

@@ -5,7 +5,7 @@ package vitastor
 
 const (
     vitastorCSIDriverName    = "csi.vitastor.io"
-    vitastorCSIDriverVersion = "0.8.9"
+    vitastorCSIDriverVersion = "0.8.3"
 )
 
 // Config struct fills the parameters of request or user input

csi/src/controllerserver.go

@@ -6,11 +6,11 @@ package vitastor
 import (
     "context"
     "encoding/json"
-    "fmt"
     "strings"
     "bytes"
     "strconv"
     "time"
+    "fmt"
     "os"
     "os/exec"
     "io/ioutil"
@@ -21,6 +21,8 @@ import (
     "google.golang.org/grpc/codes"
     "google.golang.org/grpc/status"
 
+    "go.etcd.io/etcd/clientv3"
+
     "github.com/container-storage-interface/spec/lib/go/csi"
 )
 
@@ -112,34 +114,6 @@ func GetConnectionParams(params map[string]string) (map[string]string, []string,
     return ctxVars, etcdUrl, etcdPrefix
 }
 
-func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
-{
-    if (ctxVars["etcdUrl"] != "")
-    {
-        args = append(args, "--etcd_address", ctxVars["etcdUrl"])
-    }
-    if (ctxVars["etcdPrefix"] != "")
-    {
-        args = append(args, "--etcd_prefix", ctxVars["etcdPrefix"])
-    }
-    if (ctxVars["configPath"] != "")
-    {
-        args = append(args, "--config_path", ctxVars["configPath"])
-    }
-    c := exec.Command("/usr/bin/vitastor-cli", args...)
-    var stdout, stderr bytes.Buffer
-    c.Stdout = &stdout
-    c.Stderr = &stderr
-    err := c.Run()
-    stderrStr := string(stderr.Bytes())
-    if (err != nil)
-    {
-        klog.Errorf("vitastor-cli %s failed: %s, status %s\n", strings.Join(args, " "), stderrStr, err)
-        return nil, status.Error(codes.Internal, stderrStr+" (status "+err.Error()+")")
-    }
-    return stdout.Bytes(), nil
-}
-
 // Create the volume
 func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error)
 {
@@ -172,41 +146,128 @@ func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVol
         volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
     }
 
-    ctxVars, etcdUrl, _ := GetConnectionParams(req.Parameters)
+    // FIXME: The following should PROBABLY be implemented externally in a management tool
+
+    ctxVars, etcdUrl, etcdPrefix := GetConnectionParams(req.Parameters)
     if (len(etcdUrl) == 0)
     {
         return nil, status.Error(codes.InvalidArgument, "no etcdUrl in storage class configuration and no etcd_address in vitastor.conf")
     }
 
-    // Create image using vitastor-cli
-    _, err := invokeCLI(ctxVars, []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) })
+    // Connect to etcd
+    cli, err := clientv3.New(clientv3.Config{
+        DialTimeout: ETCD_TIMEOUT,
+        Endpoints: etcdUrl,
+    })
     if (err != nil)
     {
-        if (strings.Index(err.Error(), "already exists") > 0)
+        return nil, status.Error(codes.Internal, "failed to connect to etcd at "+strings.Join(etcdUrl, ",")+": "+err.Error())
+    }
+    defer cli.Close()
+
+    var imageId uint64 = 0
+    for
+    {
+        // Check if the image exists
+        ctx, cancel := context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+        resp, err := cli.Get(ctx, etcdPrefix+"/index/image/"+volName)
+        cancel()
+        if (err != nil)
         {
-            stat, err := invokeCLI(ctxVars, []string{ "ls", "--json", volName })
+            return nil, status.Error(codes.Internal, "failed to read key from etcd: "+err.Error())
+        }
+        if (len(resp.Kvs) > 0)
+        {
+            kv := resp.Kvs[0]
+            var v InodeIndex
+            err := json.Unmarshal(kv.Value, &v)
             if (err != nil)
             {
-                return nil, err
+                return nil, status.Error(codes.Internal, "invalid /index/image/"+volName+" key in etcd: "+err.Error())
             }
-            var inodeCfg []InodeConfig
-            err = json.Unmarshal(stat, &inodeCfg)
+            poolId = v.PoolId
+            imageId = v.Id
+            inodeCfgKey := fmt.Sprintf("/config/inode/%d/%d", poolId, imageId)
+            ctx, cancel := context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+            resp, err := cli.Get(ctx, etcdPrefix+inodeCfgKey)
+            cancel()
             if (err != nil)
             {
-                return nil, status.Error(codes.Internal, "Invalid JSON in vitastor-cli ls: "+err.Error())
+                return nil, status.Error(codes.Internal, "failed to read key from etcd: "+err.Error())
             }
-            if (len(inodeCfg) == 0)
+            if (len(resp.Kvs) == 0)
             {
-                return nil, status.Error(codes.Internal, "vitastor-cli create said that image already exists, but ls can't find it")
+                return nil, status.Error(codes.Internal, "missing "+inodeCfgKey+" key in etcd")
             }
-            if (inodeCfg[0].Size < uint64(volSize))
+            var inodeCfg InodeConfig
+            err = json.Unmarshal(resp.Kvs[0].Value, &inodeCfg)
+            if (err != nil)
+            {
+                return nil, status.Error(codes.Internal, "invalid "+inodeCfgKey+" key in etcd: "+err.Error())
+            }
+            if (inodeCfg.Size < uint64(volSize))
             {
                 return nil, status.Error(codes.Internal, "image "+volName+" is already created, but size is less than expected")
             }
         }
         else
         {
-            return nil, err
+            // Find a free ID
+            // Create image metadata in a transaction verifying that the image doesn't exist yet AND ID is still free
+            maxIdKey := fmt.Sprintf("%s/index/maxid/%d", etcdPrefix, poolId)
+            ctx, cancel := context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+            resp, err := cli.Get(ctx, maxIdKey)
+            cancel()
+            if (err != nil)
+            {
+                return nil, status.Error(codes.Internal, "failed to read key from etcd: "+err.Error())
+            }
+            var modRev int64
+            var nextId uint64
+            if (len(resp.Kvs) > 0)
+            {
+                var err error
+                nextId, err = strconv.ParseUint(string(resp.Kvs[0].Value), 10, 64)
+                if (err != nil)
+                {
+                    return nil, status.Error(codes.Internal, maxIdKey+" contains invalid ID")
+                }
+                modRev = resp.Kvs[0].ModRevision
+                nextId++
+            }
+            else
+            {
+                nextId = 1
+            }
+            inodeIdxJson, _ := json.Marshal(InodeIndex{
+                Id: nextId,
+                PoolId: poolId,
+            })
+            inodeCfgJson, _ := json.Marshal(InodeConfig{
+                Name: volName,
+                Size: uint64(volSize),
+            })
+            ctx, cancel = context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+            txnResp, err := cli.Txn(ctx).If(
+                clientv3.Compare(clientv3.ModRevision(fmt.Sprintf("%s/index/maxid/%d", etcdPrefix, poolId)), "=", modRev),
+                clientv3.Compare(clientv3.CreateRevision(fmt.Sprintf("%s/index/image/%s", etcdPrefix, volName)), "=", 0),
+                clientv3.Compare(clientv3.CreateRevision(fmt.Sprintf("%s/config/inode/%d/%d", etcdPrefix, poolId, nextId)), "=", 0),
+            ).Then(
+                clientv3.OpPut(fmt.Sprintf("%s/index/maxid/%d", etcdPrefix, poolId), fmt.Sprintf("%d", nextId)),
+                clientv3.OpPut(fmt.Sprintf("%s/index/image/%s", etcdPrefix, volName), string(inodeIdxJson)),
+                clientv3.OpPut(fmt.Sprintf("%s/config/inode/%d/%d", etcdPrefix, poolId, nextId), string(inodeCfgJson)),
+            ).Commit()
+            cancel()
+            if (err != nil)
+            {
+                return nil, status.Error(codes.Internal, "failed to commit transaction in etcd: "+err.Error())
+            }
+            if (txnResp.Succeeded)
+            {
+                imageId = nextId
+                break
+            }
+            // Start over if the transaction fails
         }
     }
 
@@ -238,12 +299,97 @@ func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVol
     }
     volName := ctxVars["name"]
 
-    ctxVars, _, _ = GetConnectionParams(ctxVars)
+    _, etcdUrl, etcdPrefix := GetConnectionParams(ctxVars)
+    if (len(etcdUrl) == 0)
+    {
+        return nil, status.Error(codes.InvalidArgument, "no etcdUrl in storage class configuration and no etcd_address in vitastor.conf")
+    }
 
-    _, err = invokeCLI(ctxVars, []string{ "rm", volName })
+    cli, err := clientv3.New(clientv3.Config{
+        DialTimeout: ETCD_TIMEOUT,
+        Endpoints: etcdUrl,
+    })
     if (err != nil)
     {
-        return nil, err
+        return nil, status.Error(codes.Internal, "failed to connect to etcd at "+strings.Join(etcdUrl, ",")+": "+err.Error())
+    }
+    defer cli.Close()
+
+    // Find inode by name
+    ctx, cancel := context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+    resp, err := cli.Get(ctx, etcdPrefix+"/index/image/"+volName)
+    cancel()
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "failed to read key from etcd: "+err.Error())
+    }
+    if (len(resp.Kvs) == 0)
+    {
+        return nil, status.Error(codes.NotFound, "volume "+volName+" does not exist")
+    }
+    var idx InodeIndex
+    err = json.Unmarshal(resp.Kvs[0].Value, &idx)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "invalid /index/image/"+volName+" key in etcd: "+err.Error())
+    }
+
+    // Get inode config
+    inodeCfgKey := fmt.Sprintf("%s/config/inode/%d/%d", etcdPrefix, idx.PoolId, idx.Id)
+    ctx, cancel = context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+    resp, err = cli.Get(ctx, inodeCfgKey)
+    cancel()
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "failed to read key from etcd: "+err.Error())
+    }
+    if (len(resp.Kvs) == 0)
+    {
+        return nil, status.Error(codes.NotFound, "volume "+volName+" does not exist")
+    }
+    var inodeCfg InodeConfig
+    err = json.Unmarshal(resp.Kvs[0].Value, &inodeCfg)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "invalid "+inodeCfgKey+" key in etcd: "+err.Error())
+    }
+
+    // Delete inode data by invoking vitastor-cli
+    args := []string{
+        "rm-data", "--etcd_address", strings.Join(etcdUrl, ","),
+        "--pool", fmt.Sprintf("%d", idx.PoolId),
+        "--inode", fmt.Sprintf("%d", idx.Id),
+    }
+    if (ctxVars["configPath"] != "")
+    {
+        args = append(args, "--config_path", ctxVars["configPath"])
+    }
+    c := exec.Command("/usr/bin/vitastor-cli", args...)
+    var stderr bytes.Buffer
+    c.Stdout = nil
+    c.Stderr = &stderr
+    err = c.Run()
+    stderrStr := string(stderr.Bytes())
+    if (err != nil)
+    {
+        klog.Errorf("vitastor-cli rm-data failed: %s, status %s\n", stderrStr, err)
+        return nil, status.Error(codes.Internal, stderrStr+" (status "+err.Error()+")")
+    }
+
+    // Delete inode config in etcd
+    ctx, cancel = context.WithTimeout(context.Background(), ETCD_TIMEOUT)
+    txnResp, err := cli.Txn(ctx).Then(
+        clientv3.OpDelete(fmt.Sprintf("%s/index/image/%s", etcdPrefix, volName)),
+        clientv3.OpDelete(fmt.Sprintf("%s/config/inode/%d/%d", etcdPrefix, idx.PoolId, idx.Id)),
+    ).Commit()
+    cancel()
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "failed to delete keys in etcd: "+err.Error())
+    }
+    if (!txnResp.Succeeded)
+    {
+        return nil, status.Error(codes.Internal, "failed to delete keys in etcd: transaction failed")
     }
 
     return &csi.DeleteVolumeResponse{}, nil

debian/changelog

@@ -1,10 +1,10 @@
-vitastor (0.8.9-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Bugfixes
 
  -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
 
-vitastor (0.8.9-1) unstable; urgency=medium
+vitastor (0.8.3-1) unstable; urgency=medium
 
   * Implement NFS proxy
   * Add documentation

debian/vitastor.Dockerfile

@@ -34,8 +34,8 @@ RUN set -e -x; \
     mkdir -p /root/packages/vitastor-$REL; \
     rm -rf /root/packages/vitastor-$REL/*; \
     cd /root/packages/vitastor-$REL; \
-    cp -r /root/vitastor vitastor-0.8.9; \
-    cd vitastor-0.8.9; \
+    cp -r /root/vitastor vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     ln -s /root/fio-build/fio-*/ ./fio; \
     FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
@@ -48,8 +48,8 @@ RUN set -e -x; \
     rm -rf a b; \
     echo "dep:fio=$FIO" > debian/fio_version; \
     cd /root/packages/vitastor-$REL; \
-    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.9.orig.tar.xz vitastor-0.8.9; \
-    cd vitastor-0.8.9; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.8.3.orig.tar.xz vitastor-0.8.3; \
+    cd vitastor-0.8.3; \
     V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
     DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
     DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \

docs/config.en.md

@@ -17,16 +17,14 @@ Configuration parameters can be set in 3 places:
 - Configuration file (`/etc/vitastor/vitastor.conf` or other path)
 - etcd key `/vitastor/config/global`. Most variables can be set there, but etcd
   connection parameters should obviously be set in the configuration file.
-- Command line of Vitastor components: OSD (when you run it without vitastor-disk),
-  mon, fio and QEMU options, OpenStack/Proxmox/etc configuration. The latter
-  doesn't allow to set all variables directly, but it allows to override the
-  configuration file and set everything you need inside it.
-- OSD superblocks created by [vitastor-disk](../usage/disk.en.md) contain
-  primarily disk layout parameters of specific OSDs. In fact, these parameters
-  are automatically passed into the command line of vitastor-osd process, so
-  they have the same "status" as command-line parameters.
+- Command line of Vitastor components: OSD, mon, fio and QEMU options,
+  OpenStack/Proxmox/etc configuration. The latter doesn't allow to set all
+  variables directly, but it allows to override the configuration file and
+  set everything you need inside it.
 
 In the future, additional configuration methods may be added:
+- OSD superblock which will, by design, contain parameters related to the disk
+  layout and to one specific OSD.
 - OSD-specific keys in etcd like `/vitastor/config/osd/<number>`.
 
 ## Parameter Reference

docs/config.ru.md

@@ -19,17 +19,14 @@
 - Ключе в etcd `/vitastor/config/global`. Большая часть параметров может
   задаваться там, кроме, естественно, самих параметров соединения с etcd,
   которые должны задаваться в файле конфигурации
-- В командной строке компонентов Vitastor: OSD (при ручном запуске без vitastor-disk),
-  монитора, опциях fio и QEMU, настроек OpenStack, Proxmox и т.п. Последние,
-  как правило, не включают полный набор параметров напрямую, но позволяют
-  определить путь к файлу конфигурации и задать любые параметры в нём.
-- В суперблоке OSD, записываемом [vitastor-disk](../usage/disk.ru.md) - параметры,
-  связанные с дисковым форматом и с этим конкретным OSD. На самом деле,
-  при запуске OSD эти параметры автоматически передаются в командную строку
-  процесса vitastor-osd, то есть по "статусу" они эквивалентны параметрам
-  командной строки OSD.
+- В командной строке компонентов Vitastor: OSD, монитора, опциях fio и QEMU,
+  настроек OpenStack, Proxmox и т.п. Последние, как правило, не включают полный
+  набор параметров напрямую, но разрешают определить путь к файлу конфигурации
+  и задать любые параметры в нём.
 
 В будущем также могут быть добавлены другие способы конфигурации:
+- Суперблок OSD, в котором будут храниться параметры OSD, связанные с дисковым
+  форматом и с этим конкретным OSD.
 - OSD-специфичные ключи в etcd типа `/vitastor/config/osd/<номер>`.
 
 ## Список параметров

docs/config/network.en.md

@@ -19,7 +19,6 @@ between clients, OSDs and etcd.
 - [rdma_max_sge](#rdma_max_sge)
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
-- [rdma_max_send](#rdma_max_send)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -75,12 +74,6 @@ to work. For example, Mellanox ConnectX-3 and older adapters don't have
 Implicit ODP, so they're unsupported by Vitastor. Run `ibv_devinfo -v` as
 root to list available RDMA devices and their features.
 
-Remember that you also have to configure your network switches if you use
-RoCE/RoCEv2, otherwise you may experience unstable performance. Refer to
-the manual of your network vendor for details about setting up the switch
-for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
-PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
-
 ## rdma_port_num
 
 - Type: integer
@@ -123,30 +116,20 @@ required to change this parameter.
 ## rdma_max_msg
 
 - Type: integer
-- Default: 132096
+- Default: 1048576
 
 Maximum size of a single RDMA send or receive operation in bytes.
 
 ## rdma_max_recv
 
-- Type: integer
-- Default: 16
-
-Maximum number of RDMA receive buffers per connection (RDMA requires
-preallocated buffers to receive data). Each buffer is `rdma_max_msg` bytes
-in size. So this setting directly affects memory usage: a single Vitastor
-RDMA client uses `rdma_max_recv * rdma_max_msg * OSD_COUNT` bytes of memory.
-Default is roughly 2 MB * number of OSDs.
-
-## rdma_max_send
-
 - Type: integer
 - Default: 8
 
-Maximum number of outstanding RDMA send operations per connection. Should be
-less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
-Doesn't affect memory usage - additional memory isn't allocated for send
-operations.
+Maximum number of parallel RDMA receive operations. Note that this number
+of receive buffers `rdma_max_msg` in size are allocated for each client,
+so this setting actually affects memory usage. This is because RDMA receive
+operations are (sadly) still not zero-copy in Vitastor. It may be fixed in
+later versions.
 
 ## peer_connect_interval
 

docs/config/network.ru.md

@@ -19,7 +19,6 @@
 - [rdma_max_sge](#rdma_max_sge)
 - [rdma_max_msg](#rdma_max_msg)
 - [rdma_max_recv](#rdma_max_recv)
-- [rdma_max_send](#rdma_max_send)
 - [peer_connect_interval](#peer_connect_interval)
 - [peer_connect_timeout](#peer_connect_timeout)
 - [osd_idle_timeout](#osd_idle_timeout)
@@ -79,13 +78,6 @@ Implicit On-Demand Paging (Implicit ODP) и Scatter/Gather (SG). Наприме
 суперпользователя, чтобы посмотреть список доступных RDMA-устройств, их
 параметры и возможности.
 
-Обратите внимание, что если вы используете RoCE/RoCEv2, вам также необходимо
-правильно настроить для него коммутаторы, иначе вы можете столкнуться с
-нестабильной производительностью. Подробную информацию о настройке
-коммутатора для RoCEv2 ищите в документации производителя. Обычно это
-подразумевает настройку сети без потерь на основе PFC (Priority Flow
-Control) и ECN (Explicit Congestion Notification).
-
 ## rdma_port_num
 
 - Тип: целое число
@@ -129,32 +121,22 @@ OSD в любом случае согласовывают реальное зн
 ## rdma_max_msg
 
 - Тип: целое число
-- Значение по умолчанию: 132096
+- Значение по умолчанию: 1048576
 
 Максимальный размер одной RDMA-операции отправки или приёма.
 
 ## rdma_max_recv
 
-- Тип: целое число
-- Значение по умолчанию: 16
-
-Максимальное число буферов для RDMA-приёма данных на одно соединение
-(RDMA требует заранее выделенных буферов для приёма данных). Каждый буфер
-имеет размер `rdma_max_msg` байт. Таким образом, настройка прямо влияет на
-потребление памяти - один Vitastor-клиент с RDMA использует
-`rdma_max_recv * rdma_max_msg * ЧИСЛО_OSD` байт памяти, по умолчанию -
-примерно 2 МБ * число OSD.
-
-## rdma_max_send
-
 - Тип: целое число
 - Значение по умолчанию: 8
 
-Максимальное число RDMA-операций отправки, отправляемых в очередь одного
-соединения. Желательно, чтобы оно было меньше `rdma_max_recv`, чтобы
-у принимающей стороны в процессе работы не заканчивались буферы на приём.
-Не влияет на потребление памяти - дополнительная память на операции отправки
-не выделяется.
+Максимальное число параллельных RDMA-операций получения данных. Следует
+иметь в виду, что данное число буферов размером `rdma_max_msg` выделяется
+для каждого подключённого клиентского соединения, так что данная настройка
+влияет на потребление памяти. Это так потому, что RDMA-приём данных в
+Vitastor, увы, всё равно не является zero-copy, т.е. всё равно 1 раз
+копирует данные в памяти. Данная особенность, возможно, будет исправлена в
+более новых версиях Vitastor.
 
 ## peer_connect_interval
 

docs/config/src/network.yml

@@ -53,12 +53,6 @@
     to work. For example, Mellanox ConnectX-3 and older adapters don't have
     Implicit ODP, so they're unsupported by Vitastor. Run `ibv_devinfo -v` as
     root to list available RDMA devices and their features.
-
-    Remember that you also have to configure your network switches if you use
-    RoCE/RoCEv2, otherwise you may experience unstable performance. Refer to
-    the manual of your network vendor for details about setting up the switch
-    for RoCEv2 correctly. Usually it means setting up Lossless Ethernet with
-    PFC (Priority Flow Control) and ECN (Explicit Congestion Notification).
   info_ru: |
     Название RDMA-устройства для связи с Vitastor OSD (например, "rocep5s0f0").
     Имейте в виду, что поддержка RDMA в Vitastor требует функций устройства
@@ -67,13 +61,6 @@
     потому не поддерживаются в Vitastor. Запустите `ibv_devinfo -v` от имени
     суперпользователя, чтобы посмотреть список доступных RDMA-устройств, их
     параметры и возможности.
-
-    Обратите внимание, что если вы используете RoCE/RoCEv2, вам также необходимо
-    правильно настроить для него коммутаторы, иначе вы можете столкнуться с
-    нестабильной производительностью. Подробную информацию о настройке
-    коммутатора для RoCEv2 ищите в документации производителя. Обычно это
-    подразумевает настройку сети без потерь на основе PFC (Priority Flow
-    Control) и ECN (Explicit Congestion Notification).
 - name: rdma_port_num
   type: int
   default: 1
@@ -127,39 +114,26 @@
     так что менять этот параметр обычно не нужно.
 - name: rdma_max_msg
   type: int
-  default: 132096
+  default: 1048576
   info: Maximum size of a single RDMA send or receive operation in bytes.
   info_ru: Максимальный размер одной RDMA-операции отправки или приёма.
 - name: rdma_max_recv
-  type: int
-  default: 16
-  info: |
-    Maximum number of RDMA receive buffers per connection (RDMA requires
-    preallocated buffers to receive data). Each buffer is `rdma_max_msg` bytes
-    in size. So this setting directly affects memory usage: a single Vitastor
-    RDMA client uses `rdma_max_recv * rdma_max_msg * OSD_COUNT` bytes of memory.
-    Default is roughly 2 MB * number of OSDs.
-  info_ru: |
-    Максимальное число буферов для RDMA-приёма данных на одно соединение
-    (RDMA требует заранее выделенных буферов для приёма данных). Каждый буфер
-    имеет размер `rdma_max_msg` байт. Таким образом, настройка прямо влияет на
-    потребление памяти - один Vitastor-клиент с RDMA использует
-    `rdma_max_recv * rdma_max_msg * ЧИСЛО_OSD` байт памяти, по умолчанию -
-    примерно 2 МБ * число OSD.
-- name: rdma_max_send
   type: int
   default: 8
   info: |
-    Maximum number of outstanding RDMA send operations per connection. Should be
-    less than `rdma_max_recv` so the receiving side doesn't run out of buffers.
-    Doesn't affect memory usage - additional memory isn't allocated for send
-    operations.
+    Maximum number of parallel RDMA receive operations. Note that this number
+    of receive buffers `rdma_max_msg` in size are allocated for each client,
+    so this setting actually affects memory usage. This is because RDMA receive
+    operations are (sadly) still not zero-copy in Vitastor. It may be fixed in
+    later versions.
   info_ru: |
-    Максимальное число RDMA-операций отправки, отправляемых в очередь одного
-    соединения. Желательно, чтобы оно было меньше `rdma_max_recv`, чтобы
-    у принимающей стороны в процессе работы не заканчивались буферы на приём.
-    Не влияет на потребление памяти - дополнительная память на операции отправки
-    не выделяется.
+    Максимальное число параллельных RDMA-операций получения данных. Следует
+    иметь в виду, что данное число буферов размером `rdma_max_msg` выделяется
+    для каждого подключённого клиентского соединения, так что данная настройка
+    влияет на потребление памяти. Это так потому, что RDMA-приём данных в
+    Vitastor, увы, всё равно не является zero-copy, т.е. всё равно 1 раз
+    копирует данные в памяти. Данная особенность, возможно, будет исправлена в
+    более новых версиях Vitastor.
 - name: peer_connect_interval
   type: sec
   min: 1

docs/installation/packages.en.md

@@ -9,7 +9,7 @@
 ## Debian
 
 - Trust Vitastor package signing key:
-  `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
+  `wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -`
 - Add Vitastor package repository to your /etc/apt/sources.list:
   - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
@@ -20,18 +20,15 @@
 ## CentOS
 
 - Add Vitastor package repository:
-  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
-  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
-  - AlmaLinux 9 and other RHEL 9 clones (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm`
+  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm`
 - Enable EPEL: `yum/dnf install epel-release`
 - Enable additional CentOS repositories:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - RHEL 9 clones: not required
 - Enable elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
-  - RHEL 9 clones: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
 - Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Installation requirements

docs/installation/packages.ru.md

@@ -9,7 +9,7 @@
 ## Debian
 
 - Добавьте ключ репозитория Vitastor:
-  `wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg`
+  `wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -`
 - Добавьте репозиторий Vitastor в /etc/apt/sources.list:
   - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main`
   - Debian 10 (Buster): `deb https://vitastor.io/debian buster main`
@@ -20,18 +20,15 @@
 ## CentOS
 
 - Добавьте в систему репозиторий Vitastor:
-  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release.rpm`
-  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release.rpm`
-  - AlmaLinux 9 и другие клоны RHEL 9 (Rocky, Oracle...): `dnf install https://vitastor.io/rpms/centos/9/vitastor-release.rpm`
+  - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm`
+  - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm`
 - Включите EPEL: `yum/dnf install epel-release`
 - Включите дополнительные репозитории CentOS:
   - CentOS 7: `yum install centos-release-scl`
   - CentOS 8: `dnf install centos-release-advanced-virtualization`
-  - Клоны RHEL 9: не нужно
 - Включите elrepo-kernel:
   - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm`
   - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm`
-  - Клоны RHEL 9: `dnf install https://www.elrepo.org/elrepo-release-9.el9.elrepo.noarch.rpm`
 - Установите пакеты: `yum/dnf install vitastor lpsolve etcd kernel-ml qemu-kvm`
 
 ## Установочные требования

docs/installation/proxmox.en.md

@@ -6,10 +6,10 @@
 
 # Proxmox VE
 
-To enable Vitastor support in Proxmox Virtual Environment (6.4-7.4 are supported):
+To enable Vitastor support in Proxmox Virtual Environment (6.4-7.3 are supported):
 
 - Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts:
-  buster for 6.4, bullseye for 7.4, pve7.1 for 7.1, pve7.2 for 7.2, pve7.3 for 7.3
+  buster for 6.4, bullseye for 7.3, pve7.1 for 7.1, pve7.2 for 7.2
 - Install vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* or see note) packages from Vitastor repository
 - Define storage in `/etc/pve/storage.cfg` (see below)
 - Block network access from VMs to Vitastor network (to OSDs and etcd),

docs/installation/proxmox.ru.md

@@ -6,10 +6,10 @@
 
 # Proxmox
 
-Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-7.4):
+Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4-7.3):
 
 - Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox:
-  buster для 6.4, bullseye для 7.4, pve7.1 для 7.1, pve7.2 для 7.2, pve7.3 для 7.3
+  buster для 6.4, bullseye для 7.3, pve7.1 для 7.1, pve7.2 для 7.2
 - Установите пакеты vitastor-client, pve-qemu-kvm, pve-storage-vitastor (* или см. сноску) из репозитория Vitastor
 - Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже)
 - Обязательно заблокируйте доступ от виртуальных машин к сети Vitastor (OSD и etcd), т.к. Vitastor (пока) не поддерживает аутентификацию

docs/intro/quickstart.en.md

@@ -45,9 +45,7 @@ On the monitor hosts:
   }
   ```
 - Initialize OSDs:
-  - SSD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. You can add
-    `--disable_data_fsync off` to leave disk cache enabled if you use desktop
-    SSDs without capacitors.
+  - SSD-only: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`
   - Hybrid, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Pass all your devices (HDD and SSD) to this script — it will partition disks and initialize journals on its own.
     This script skips HDDs which are already partitioned so if you want to use non-empty disks for
@@ -55,9 +53,7 @@ On the monitor hosts:
     but some free unpartitioned space must be available because the script creates new partitions for journals.
 - You can change OSD configuration in units or in `vitastor.conf`.
   Check [Configuration Reference](../config.en.md) for parameter descriptions.
-- If all your drives have capacitors, and even if not, but if you ran `vitastor-disk`
-  without `--disable_data_fsync off` at the first step, then put the following
-  setting into etcd: \
+- If all your drives have capacitors, create global configuration in etcd: \
   `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
 - Start all OSDs: `systemctl start vitastor.target`
 
@@ -74,15 +70,11 @@ For EC pools the configuration should look like the following:
 
 ```
 etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
-  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}}'
+  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}'
 ```
 
 After you do this, one of the monitors will configure PGs and OSDs will start them.
 
-If you use HDDs you should also add `"block_size": 1048576` to pool configuration.
-The other option is to add it into /vitastor/config/global, in this case it will
-apply to all pools by default.
-
 ## Check cluster status
 
 `vitastor-cli status`

docs/intro/quickstart.ru.md

@@ -45,9 +45,7 @@
   }
   ```
 - Инициализуйте OSD:
-  - SSD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`. Если вы используете
-    десктопные SSD без конденсаторов, можете оставить кэш включённым, добавив
-    опцию `--disable_data_fsync off`.
+  - SSD: `vitastor-disk prepare /dev/sdXXX [/dev/sdYYY ...]`
   - Гибридные, SSD+HDD: `vitastor-disk prepare --hybrid /dev/sdXXX [/dev/sdYYY ...]`.
     Передайте все ваши SSD и HDD скрипту в командной строке подряд, скрипт автоматически выделит
     разделы под журналы на SSD и данные на HDD. Скрипт пропускает HDD, на которых уже есть разделы
@@ -56,11 +54,8 @@
     для журналов, на SSD должно быть доступно свободное нераспределённое место.
 - Вы можете менять параметры OSD в юнитах systemd или в `vitastor.conf`. Описания параметров
   смотрите в [справке по конфигурации](../config.ru.md).
-- Если все ваши диски - серверные с конденсаторами, и даже если нет, но при этом
-  вы не добавляли опцию `--disable_data_fsync off` на первом шаге, а `vitastor-disk`
-  не ругался на невозможность отключения кэша дисков, пропишите следующую настройку
-  в глобальную конфигурацию в etcd: \
-  `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`.
+- Если все ваши диски - серверные с конденсаторами, пропишите это в глобальную конфигурацию в etcd: \
+  `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'`
 - Запустите все OSD: `systemctl start vitastor.target`
 
 ## Создайте пул
@@ -76,15 +71,11 @@ etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool",
 
 ```
 etcdctl --endpoints=... put /vitastor/config/pools '{"2":{"name":"ecpool",
-  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}}'
+  "scheme":"ec","pg_size":4,"parity_chunks":2,"pg_minsize":2,"pg_count":256,"failure_domain":"host"}'
 ```
 
 После этого один из мониторов должен сконфигурировать PG, а OSD должны запустить их.
 
-Если вы используете HDD-диски, то добавьте в конфигурацию пулов опцию `"block_size": 1048576`.
-Также эту опцию можно добавить в /vitastor/config/global, в этом случае она будет
-применяться ко всем пулам по умолчанию.
-
 ## Проверьте состояние кластера
 
 `vitastor-cli status`

docs/performance/theoretical.en.md

@@ -35,24 +35,15 @@ Write amplification for 4 KB blocks is usually 3-5 in Vitastor:
 If you manage to get an SSD which handles 512 byte blocks well (Optane?) you may
 lower 1, 3 and 4 to 512 bytes (1/8 of data size) and get WA as low as 2.375.
 
-Implemented NVDIMM support can basically eliminate WA at all - all extra writes will
-go to DRAM memory. But this requires a test cluster with NVDIMM - please contact me
-if you want to provide me with such cluster for tests.
-
 Lazy fsync also reduces WA for parallel workloads because journal blocks are only
 written when they fill up or fsync is requested.
 
 ## In Practice
 
-In practice, using tests from [Understanding Performance](understanding.en.md), decent TCP network,
-good server-grade SSD/NVMe drives and disabled CPU power saving, you should head for:
+In practice, using tests from [Understanding Performance](understanding.en.md)
+and good server-grade SSD/NVMe drives, you should head for:
 - At least 5000 T1Q1 replicated read and write iops (maximum 0.2ms latency)
-- At least 5000 T1Q1 EC read IOPS and at least 2200 EC write IOPS (maximum 0.45ms latency)
 - At least ~80k parallel read iops or ~30k write iops per 1 core (1 OSD)
 - Disk-speed or wire-speed linear reads and writes, whichever is the bottleneck in your case
 
 Lower results may mean that you have bad drives, bad network or some kind of misconfiguration.
-
-Current latency records:
-- 9668 T1Q1 replicated write iops (0.103 ms latency) with TCP and NVMe
-- 9143 T1Q1 replicated read iops (0.109 ms latency) with TCP and NVMe

docs/performance/theoretical.ru.md

@@ -36,25 +36,6 @@ WA (мультипликатор записи) для 4 КБ блоков в Vit
 Если вы найдёте SSD, хорошо работающий с 512-байтными блоками данных (Optane?),
 то 1, 3 и 4 можно снизить до 512 байт (1/8 от размера данных) и получить WA всего 2.375.
 
-Если реализовать поддержку NVDIMM, то WA можно, условно говоря, ликвидировать вообще - все
-дополнительные операции записи смогут обслуживаться DRAM памятью. Но для этого необходим
-тестовый кластер с NVDIMM - пишите, если готовы предоставить такой для тестов.
-
 Кроме того, WA снижается при использовании отложенного/ленивого сброса при параллельной
 нагрузке, т.к. блоки журнала записываются на диск только когда они заполняются или явным
 образом запрашивается fsync.
-
-## На практике
-
-На практике, используя тесты fio со страницы [Понимание сути производительности систем хранения](understanding.ru.md),
-нормальную TCP-сеть, хорошие серверные SSD/NVMe, при отключённом энергосбережении процессоров вы можете рассчитывать на:
-- От 5000 IOPS в 1 поток (T1Q1) и на чтение, и на запись при использовании репликации (задержка до 0.2мс)
-- От 5000 IOPS в 1 поток (T1Q1) на чтение и 2200 IOPS в 1 поток на запись при использовании EC (задержка до 0.45мс)
-- От 80000 IOPS на чтение в параллельном режиме на 1 ядро, от 30000 IOPS на запись на 1 ядро (на 1 OSD)
-- Скорость параллельного линейного чтения и записи, равная меньшему значению из скорости дисков или сети
-
-Худшие результаты означают, что у вас либо медленные диски, либо медленная сеть, либо что-то неправильно настроено.
-
-Зафиксированный на данный момент рекорд задержки:
-- 9668 IOPS (0.103 мс задержка) в 1 поток (T1Q1) на запись с TCP и NVMe при использовании репликации
-- 9143 IOPS (0.109 мс задержка) в 1 поток (T1Q1) на чтение с TCP и NVMe при использовании репликации

docs/usage/cli.en.md

@@ -14,7 +14,6 @@ It supports the following commands:
 - [df](#df)
 - [ls](#ls)
 - [create](#create)
-- [snap-create](#create)
 - [modify](#modify)
 - [rm](#rm)
 - [flatten](#flatten)
@@ -124,8 +123,6 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 
 Create a snapshot of image `<name>` (either form can be used). May be used live if only a single writer is active.
 
-See also about [how to export snapshots](qemu.en.md#exporting-snapshots).
-
 ## modify
 
 `vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force]`

docs/usage/cli.ru.md

@@ -15,7 +15,6 @@ vitastor-cli - интерфейс командной строки для адм
 - [df](#df)
 - [ls](#ls)
 - [create](#create)
-- [snap-create](#create)
 - [modify](#modify)
 - [rm](#rm)
 - [flatten](#flatten)
@@ -127,8 +126,6 @@ vitastor-cli snap-create [-p|--pool <id|name>] <image>@<snapshot>
 Создать снимок образа `<name>` (можно использовать любую форму команды). Снимок можно создавать без остановки
 клиентов, если пишущий клиент максимум 1.
 
-Смотрите также информацию о том, [как экспортировать снимки](qemu.ru.md#экспорт-снимков).
-
 ## modify
 
 `vitastor-cli modify <name> [--rename <new-name>] [--resize <size>] [--readonly | --readwrite] [-f|--force]`

docs/usage/disk.en.md

@@ -1,4 +1,4 @@
-[Documentation](../../README.md#documentation) → Usage → Disk management tool
+[Documentation](../../README.md#documentation) → Usage → Disk Tool
 
 -----
 

docs/usage/disk.ru.md

@@ -1,4 +1,4 @@
-[Документация](../../README-ru.md#документация) → Использование → Инструмент управления дисками
+[Документация](../../README-ru.md#документация) → Использование → Управление дисками
 
 -----
 

docs/usage/qemu.en.md

@@ -46,40 +46,3 @@ qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=192.168.7
 
 You can also specify `:pool=<POOL>:inode=<INODE>:size=<SIZE>` instead of `:image=<IMAGE>`
 if you don't want to use inode metadata.
-
-### Exporting snapshots
-
-Starting with 0.8.4, you can also export individual layers (snapshot diffs) using `qemu-img`.
-
-Suppose you have an image `testimg` and a snapshot `testimg@0` created with `vitastor-cli snap-create testimg@0`.
-
-Then you can export the `testimg@0` snapshot and the data written to `testimg` after creating
-the snapshot separately using the following commands (key points are using `skip-parents=1` and
-`-B backing_file` option):
-
-```
-qemu-img convert -f raw 'vitastor:etcd_host=192.168.7.2\:2379/v3:image=testimg@0' \
-    -O qcow2 testimg_0.qcow2
-
-qemu-img convert -f raw 'vitastor:etcd_host=192.168.7.2\:2379/v3:image=testimg:skip-parents=1' \
-    -O qcow2 -o 'cluster_size=4k' -B testimg_0.qcow2 testimg.qcow2
-```
-
-In fact, with `cluster_size=4k` any QCOW2 file can be used instead `-B testimg_0.qcow2`, even an empty one.
-
-QCOW2 `cluster_size=4k` option is required if you want `testimg.qcow2` to contain only the data
-overwritten  **exactly** in the child layer. With the default 64 KB QCOW2 cluster size you'll
-get a bit of extra data from parent layers, i.e. a 4 KB overwrite will result in `testimg.qcow2`
-containing 64 KB of data. And this extra data will be taken by `qemu-img` from the file passed
-in `-B` option, so you really need 4 KB cluster if you use an empty image in `-B`.
-
-After this procedure you'll get two chained QCOW2 images. To detach `testimg.qcow2` from
-its parent, run:
-
-```
-qemu-img rebase -u -b '' testimg.qcow2
-```
-
-This can be used for backups. Just note that exporting an image that is currently being written to
-is of course unsafe and doesn't produce a consistent result, so only export snapshots if you do this
-on a live VM.

docs/usage/qemu.ru.md

@@ -50,40 +50,3 @@ qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.
 
 Если вы не хотите обращаться к образу по имени, вместо `:image=<IMAGE>` можно указать номер пула, номер инода и размер:
 `:pool=<POOL>:inode=<INODE>:size=<SIZE>`.
-
-### Экспорт снимков
-
-Начиная с 0.8.4 вы можете экспортировать отдельные слои (изменения в снимках) с помощью `qemu-img`.
-
-Допустим, что у вас есть образ `testimg` и его снимок `testimg@0`, созданный с помощью `vitastor-cli snap-create testimg@0`.
-
-Тогда вы можете выгрузить снимок `testimg@0` и данные, изменённые в `testimg` после создания снимка, отдельно,
-с помощью следующих команд (ключевые моменты - использование `skip-parents=1` и опции `-B backing_file.qcow2`):
-
-```
-qemu-img convert -f raw 'vitastor:etcd_host=192.168.7.2\:2379/v3:image=testimg@0' \
-    -O qcow2 testimg_0.qcow2
-
-qemu-img convert -f raw 'vitastor:etcd_host=192.168.7.2\:2379/v3:image=testimg:skip-parents=1' \
-    -O qcow2 -o 'cluster_size=4k' -B testimg_0.qcow2 testimg.qcow2
-```
-
-На самом деле, с `cluster_size=4k` вместо `-B testimg_0.qcow2` можно использовать любой qcow2-файл,
-даже пустой.
-
-Опция QCOW2 `cluster_size=4k` нужна, если вы хотите, чтобы `testimg.qcow2` содержал **в точности**
-данные, перезаписанные в дочернем слое. С размером кластера QCOW2 по умолчанию, составляющим 64 КБ,
-вы получите немного "лишних" данных из родительских слоёв - перезапись 4 КБ будет приводить к тому,
-что в `testimg.qcow2` будет появляться 64 КБ данных. Причём "лишние" данные qemu-img будет брать
-как раз из файла, указанного в опции `-B`, так что если там указан пустой образ, кластер обязан быть 4 КБ.
-
-После данной процедуры вы получите два QCOW2-образа, связанных в цепочку. Чтобы "отцепить" образ
-`testimg.qcow2` от базового, выполните:
-
-```
-qemu-img rebase -u -b '' testimg.qcow2
-```
-
-Это можно использовать для резервного копирования. Только помните, что экспортировать образ, в который
-в то же время идёт запись, небезопасно - результат чтения не будет целостным. Так что если вы работаете
-с активными виртуальными машинами, экспортируйте только их снимки, но не сам образ.

mon/PGUtil.js

@@ -43,16 +43,16 @@ function finish_pg_history(merged_history)
     merged_history.all_peers = Object.values(merged_history.all_peers);
 }
 
-function scale_pg_count(prev_pgs, real_prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
+function scale_pg_count(prev_pgs, prev_pg_history, new_pg_history, new_pg_count)
 {
-    const old_pg_count = real_prev_pgs.length;
+    const old_pg_count = prev_pgs.length;
     // Add all possibly intersecting PGs to the history of new PGs
     if (!(new_pg_count % old_pg_count))
     {
         // New PG count is a multiple of old PG count
         for (let i = 0; i < new_pg_count; i++)
         {
-            add_pg_history(new_pg_history, i, real_prev_pgs, prev_pg_history, i % old_pg_count);
+            add_pg_history(new_pg_history, i, prev_pgs, prev_pg_history, i % old_pg_count);
             finish_pg_history(new_pg_history[i]);
         }
     }
@@ -64,7 +64,7 @@ function scale_pg_count(prev_pgs, real_prev_pgs, prev_pg_history, new_pg_history
         {
             for (let j = 0; j < mul; j++)
             {
-                add_pg_history(new_pg_history, i, real_prev_pgs, prev_pg_history, i+j*new_pg_count);
+                add_pg_history(new_pg_history, i, prev_pgs, prev_pg_history, i+j*new_pg_count);
             }
             finish_pg_history(new_pg_history[i]);
         }
@@ -76,7 +76,7 @@ function scale_pg_count(prev_pgs, real_prev_pgs, prev_pg_history, new_pg_history
         let merged_history = {};
         for (let i = 0; i < old_pg_count; i++)
         {
-            add_pg_history(merged_history, 1, real_prev_pgs, prev_pg_history, i);
+            add_pg_history(merged_history, 1, prev_pgs, prev_pg_history, i);
         }
         finish_pg_history(merged_history[1]);
         for (let i = 0; i < new_pg_count; i++)
@@ -90,15 +90,15 @@ function scale_pg_count(prev_pgs, real_prev_pgs, prev_pg_history, new_pg_history
         new_pg_history[i] = null;
     }
     // Just for the lp_solve optimizer - pick a "previous" PG for each "new" one
-    if (prev_pgs.length < new_pg_count)
+    if (old_pg_count < new_pg_count)
     {
-        for (let i = prev_pgs.length; i < new_pg_count; i++)
+        for (let i = old_pg_count; i < new_pg_count; i++)
         {
-            prev_pgs[i] = prev_pgs[i % prev_pgs.length];
+            prev_pgs[i] = prev_pgs[i % old_pg_count];
         }
     }
-    else if (prev_pgs.length > new_pg_count)
+    else if (old_pg_count > new_pg_count)
     {
-        prev_pgs.splice(new_pg_count, prev_pgs.length-new_pg_count);
+        prev_pgs.splice(new_pg_count, old_pg_count-new_pg_count);
     }
 }

mon/lp-optimizer.js

@@ -50,8 +50,7 @@ async function lp_solve(text)
     return { score, vars };
 }
 
-async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, hier_sizes = null,
-    max_combinations = 10000, parity_space = 1, ordered = false, seq_layout = false })
+async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
 {
     if (!pg_count || !osd_tree)
     {
@@ -59,7 +58,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
     }
     const all_weights = Object.assign({}, ...Object.values(osd_tree));
     const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
-    const all_pgs = Object.values(random_hier_combinations(osd_tree, hier_sizes || [ pg_size, 1 ], max_combinations, parity_space > 1, seq_layout));
+    const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
     const pg_per_osd = {};
     for (const pg of all_pgs)
     {
@@ -217,45 +216,39 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
     return move_weights;
 }
 
-function build_parent_per_leaf(osd_tree, res = {}, parents = [])
-{
-    for (const item in osd_tree)
-    {
-        if (osd_tree[item] instanceof Object)
-            build_parent_per_leaf(osd_tree[item], res, [ ...parents, item ]);
-        else
-            res[item] = parents;
-    }
-    return res;
-}
-
-function add_valid_previous(osd_tree, prev_weights, all_pgs, hier_sizes)
+function add_valid_previous(osd_tree, prev_weights, all_pgs)
 {
     // Add previous combinations that are still valid
-    const parent_per_osd = build_parent_per_leaf(osd_tree);
+    const hosts = Object.keys(osd_tree).sort();
+    const host_per_osd = {};
+    for (const host in osd_tree)
+    {
+        for (const osd in osd_tree[host])
+        {
+            host_per_osd[osd] = host;
+        }
+    }
     skip_pg: for (const pg_name in prev_weights)
     {
-        const seen = [];
+        const seen_hosts = {};
         const pg = pg_name.substr(3).split(/_/);
         for (const osd of pg)
         {
-            if (!parent_per_osd[osd])
-                continue skip_pg;
-            for (let i = 0; i < parent_per_osd[osd].length; i++)
+            if (!host_per_osd[osd] || seen_hosts[host_per_osd[osd]])
             {
-                seen[parent_per_osd[osd][i]]++;
-                if (seen[parent_per_osd[osd][i]] > hier_sizes[i])
-                    continue skip_pg;
+                continue skip_pg;
             }
+            seen_hosts[host_per_osd[osd]] = true;
         }
         if (!all_pgs[pg_name])
+        {
             all_pgs[pg_name] = pg;
+        }
     }
 }
 
 // Try to minimize data movement
-async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2,
-    hier_sizes = null, max_combinations = 10000, parity_space = 1, ordered = false, seq_layout = false })
+async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
 {
     if (!osd_tree)
     {
@@ -280,10 +273,10 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
     }
     const old_pg_size = prev_int_pgs[0].length;
     // Get all combinations
-    let all_pgs = random_hier_combinations(osd_tree, hier_sizes || [ pg_size, 1 ], max_combinations, parity_space > 1, seq_layout);
+    let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
     if (old_pg_size == pg_size)
     {
-        add_valid_previous(osd_tree, prev_weights, all_pgs, hier_sizes || [ pg_size, 1 ]);
+        add_valid_previous(osd_tree, prev_weights, all_pgs);
     }
     all_pgs = Object.values(all_pgs);
     const pg_per_osd = {};
@@ -509,147 +502,41 @@ function put_aligned_pgs(aligned_pgs, int_pgs, prev_int_pgs, keygen)
     }
 }
 
-// Convert multi-level tree_node = { level: number|string, id?: string, size?: number, children?: tree_node[] }
+// Convert multi-level osd_tree = { level: number|string, id?: string, size?: number, children?: osd_tree }[]
 // levels = { string: number }
-// to a multi-level OSD tree suitable for random_hier_combinations()
-// (or in case of just 2 levels - for all_combinations() / random_combinations())
-//
-// Example:
-// tree_node = { level: 'dc', children: [ { level: 'rack', children: [ { level: 'host', children: [ { level: 'osd', size: 10 } ] } ] } ] }
-// extract_levels = [ 'rack', 'osd' ]
-// level_defs = { dc: 1, rack: 2, host: 3, osd: 4 }
-//
-// Result:
-// { rack0: { osd1: 10 } }
-function extract_tree_levels(tree_node, extract_levels, level_defs, new_tree = { idx: 1, items: {} })
+// to a two-level osd_tree suitable for all_combinations()
+function flatten_tree(osd_tree, levels, failure_domain_level, osd_level, domains = {}, i = { i: 1 })
 {
-    const next_level = Number(level_defs[extract_levels[0]] || extract_levels[0]) || 0;
-    const level_name = level_defs[extract_levels[0]] ? extract_levels[0] : 'l'+extract_levels[0]+'_';
-    const is_leaf = extract_levels.length == 1;
-    if ((level_defs[tree_node.level] || tree_node.level) >= next_level)
+    osd_level = levels[osd_level] || osd_level;
+    failure_domain_level = levels[failure_domain_level] || failure_domain_level;
+    for (const node of osd_tree)
     {
-        if (!is_leaf)
+        if ((levels[node.level] || node.level) < failure_domain_level)
         {
-            // Insert a (possibly fake) level
-            const nt = { idx: 1, items: {} };
-            new_tree.items[level_name+(new_tree.idx++)] = nt.items;
-            extract_tree_levels(tree_node, extract_levels.slice(1), level_defs, nt);
+            flatten_tree(node.children||[], levels, failure_domain_level, osd_level, domains, i);
         }
         else
         {
-            // Insert a leaf node
-            const leaf_id = tree_node.id || (level_name+(new_tree.idx++));
-            new_tree.items[leaf_id] = tree_node.size;
+            domains['dom'+(i.i++)] = extract_osds([ node ], levels, osd_level);
         }
     }
-    else
-    {
-        for (const child_node of tree_node.children||[])
-        {
-            extract_tree_levels(child_node, extract_levels, level_defs, new_tree);
-        }
-    }
-    return new_tree.items;
+    return domains;
 }
 
-// generate random PGs with hierarchical failure domains, i.e. for example 3 DC each with 2 HOSTS
-// osd_tree = { level3_id: { level2_id: { level1_id: scalar_value } }, ... }
-//   osd_tree may contain arbitrary number of levels, but level count must be the same across the whole tree
-// size_per_level = number of items to select on each level, for example [3, 2, 1].
-//   must have the same number of items as the osd_tree level count.
-// count = PG count to generate
-// ordered = don't treat (x,y) and (y,x) as equal
-// seq_layout = true for the [DC1,DC1,DC2,DC2,DC3,DC3] layout, false for [DC1,DC2,DC3,DC1,DC2,DC3] layout
-function random_hier_combinations(osd_tree, size_per_level, count, ordered, seq_layout)
+function extract_osds(osd_tree, levels, osd_level, osds = {})
 {
-    let seed = 0x5f020e43;
-    const rng = () =>
+    for (const node of osd_tree)
     {
-        seed ^= seed << 13;
-        seed ^= seed >> 17;
-        seed ^= seed << 5;
-        return seed + 2147483648;
-    };
-    const get_max_level = (o) =>
-    {
-        let lvl = 0;
-        while (o instanceof Object)
+        if ((levels[node.level] || node.level) >= osd_level)
         {
-            for (const k in o)
-            {
-                lvl++;
-                o = o[k];
-                break;
-            }
+            osds[node.id] = node.size;
         }
-        return lvl;
-    };
-    const max_level = get_max_level(osd_tree);
-    const gen_pg = (select) =>
-    {
-        let pg = [ osd_tree ];
-        for (let level = 0; level < max_level; level++)
+        else
         {
-            let npg = [];
-            for (let i = 0; i < pg.length; i++)
-            {
-                const keys = pg[i] instanceof Object ? Object.keys(pg[i]) : [];
-                const max_keys = keys.length < size_per_level[level] ? keys.length : size_per_level[level];
-                for (let j = 0; j < max_keys; j++)
-                {
-                    const r = select(level, i, j, (ordered ? keys.length : (keys.length - (max_keys - j - 1))));
-                    const el = pg[i][keys[r]] instanceof Object ? pg[i][keys[r]] : keys[r];
-                    npg[seq_layout ? i*size_per_level[level]+j : j*pg.length+i] = el;
-                    keys.splice(ordered ? r : 0, ordered ? 1 : (r+1));
-                }
-                for (let j = max_keys; j < size_per_level[level]; j++)
-                    npg[seq_layout ? i*size_per_level[level]+j : j*pg.length+i] = NO_OSD;
-            }
-            pg = npg;
+            extract_osds(node.children||[], levels, osd_level, osds);
         }
-        return pg;
-    };
-    const r = {};
-    // Generate random combinations including each OSD at least once
-    let has_next = true;
-    let ctr = [];
-    while (has_next)
-    {
-        let pg = gen_pg((level, i, j, n) =>
-        {
-            if (i == 0 && j == 0)
-            {
-                // Select a pre-determined OSD in the first position on each level
-                const r = ctr[level] == null || ctr[level][1] != n ? 0 : ctr[level][0];
-                ctr[level] = [ r, n ];
-                return r;
-            }
-            return rng() % n;
-        });
-        for (let i = ctr.length-1; i >= 0; i--)
-        {
-            ctr[i][0]++;
-            if (ctr[i][0] < ctr[i][1])
-                break;
-            else
-                ctr[i] = null;
-        }
-        has_next = ctr[0] != null;
-        const cyclic_pgs = [ pg ];
-        if (ordered)
-            for (let i = 1; i < pg.size; i++)
-                cyclic_pgs.push([ ...pg.slice(i), ...pg.slice(0, i) ]);
-        for (const pg of cyclic_pgs)
-            r['pg_'+pg.join('_')] = pg;
     }
-    // Generate purely random combinations
-    while (count > 0)
-    {
-        let pg = gen_pg((l, i, j, n) => rng() % n);
-        r['pg_'+pg.join('_')] = pg;
-        count--;
-    }
-    return r;
+    return osds;
 }
 
 // ordered = don't treat (x,y) and (y,x) as equal
@@ -663,8 +550,8 @@ function random_combinations(osd_tree, pg_size, count, ordered)
         seed ^= seed << 5;
         return seed + 2147483648;
     };
+    const hosts = Object.keys(osd_tree).sort();
     const osds = Object.keys(osd_tree).reduce((a, c) => { a[c] = Object.keys(osd_tree[c]).sort(); return a; }, {});
-    const hosts = Object.keys(osd_tree).sort().filter(h => osds[h].length > 0);
     const r = {};
     // Generate random combinations including each OSD at least once
     for (let h = 0; h < hosts.length; h++)
@@ -865,12 +752,11 @@ module.exports = {
     pg_weights_space_efficiency,
     pg_list_space_efficiency,
     pg_per_osd_space_efficiency,
-    extract_tree_levels,
+    flatten_tree,
 
     lp_solve,
     make_int_pgs,
     align_pgs,
     random_combinations,
-    random_hier_combinations,
     all_combinations,
 };

mon/make-etcd

@@ -79,7 +79,7 @@ StartLimitInterval=0
 RestartSec=10
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=local.target
 `);
     await system(`useradd etcd`);
     await system(`systemctl daemon-reload`);

mon/mon.js

@@ -51,9 +51,8 @@ const etcd_tree = {
             // THIS IS JUST A POOR MAN'S CONFIG DOCUMENTATION
             // etcd connection
             config_path: "/etc/vitastor/vitastor.conf",
-            etcd_prefix: "/vitastor",
-            // etcd connection - configurable online
             etcd_address: "10.0.115.10:2379/v3",
+            etcd_prefix: "/vitastor",
             // mon
             etcd_mon_ttl: 30, // min: 10
             etcd_mon_timeout: 1000, // ms. min: 0
@@ -71,15 +70,14 @@ const etcd_tree = {
             rdma_gid_index: 0,
             rdma_mtu: 4096,
             rdma_max_sge: 128,
-            rdma_max_send: 8,
-            rdma_max_recv: 16,
-            rdma_max_msg: 132096,
+            rdma_max_send: 32,
+            rdma_max_recv: 8,
+            rdma_max_msg: 1048576,
+            log_level: 0,
             block_size: 131072,
             disk_alignment: 4096,
             bitmap_granularity: 4096,
             immediate_commit: false, // 'all' or 'small'
-            // client and osd - configurable online
-            log_level: 0,
             client_dirty_limit: 33554432,
             peer_connect_interval: 5, // seconds. min: 1
             peer_connect_timeout: 5, // seconds. min: 1
@@ -97,19 +95,18 @@ const etcd_tree = {
             osd_network: null, // "192.168.7.0/24" or an array of masks
             bind_address: "0.0.0.0",
             bind_port: 0,
-            readonly: false,
-            osd_memlock: false,
-            // osd - configurable online
             autosync_interval: 5,
             autosync_writes: 128,
             client_queue_depth: 128, // unused
             recovery_queue_depth: 4,
             recovery_sync_batch: 16,
+            readonly: false,
             no_recovery: false,
             no_rebalance: false,
             print_stats_interval: 3,
             slow_log_interval: 10,
             inode_vanish_time: 60,
+            osd_memlock: false,
             // blockstore - fixed in superblock
             block_size,
             disk_alignment,
@@ -128,15 +125,14 @@ const etcd_tree = {
             meta_offset,
             disable_meta_fsync,
             disable_device_lock,
-            // blockstore - configurable offline
+            // blockstore - configurable
+            max_write_iodepth,
+            min_flusher_count: 1,
+            max_flusher_count: 256,
             inmemory_metadata,
             inmemory_journal,
             journal_sector_buffer_count,
             journal_no_same_sector_overwrites,
-            // blockstore - configurable online
-            max_write_iodepth,
-            min_flusher_count: 1,
-            max_flusher_count: 256,
             throttle_small_writes: false,
             throttle_target_iops: 100,
             throttle_target_mbs: 100,
@@ -159,10 +155,6 @@ const etcd_tree = {
                 // number of parity chunks, required for EC
                 parity_chunks?: 1,
                 pg_count: 100,
-                // failure_domain = string | { string: int }
-                // the second case specifies multiple failure domains. example:
-                // { datacenter: 3, host: 2 } - means 3 datacenters with 2 hosts each, for EC 4+2
-                // guarantees availability on outage of either 1 datacenter or 2 hosts
                 failure_domain: 'host',
                 max_osd_combinations: 10000,
                 // block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
@@ -269,7 +261,7 @@ const etcd_tree = {
             /* <pool_id>: {
                 <pg_id>: {
                     primary: osd_num_t,
-                    state: ("starting"|"peering"|"incomplete"|"active"|"repeering"|"stopping"|"offline"|
+                    state: ("starting"|"peering"|"peered"|"incomplete"|"active"|"repeering"|"stopping"|"offline"|
                         "degraded"|"has_incomplete"|"has_degraded"|"has_misplaced"|"has_unclean"|
                         "has_invalid"|"left_on_dead")[],
                 }
@@ -852,7 +844,7 @@ class Mon
         }
         for (const node_id in tree)
         {
-            if (node_id === '' || tree[node_id].level === 'osd' && (!tree[node_id].size || tree[node_id].size <= 0))
+            if (node_id === '')
             {
                 continue;
             }
@@ -960,9 +952,9 @@ class Mon
         return alive_set[this.rng() % alive_set.length];
     }
 
-    save_new_pgs_txn(save_to, request, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history)
+    save_new_pgs_txn(request, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history)
     {
-        const aff_osds = this.get_affinity_osds(this.state.config.pools[pool_id] || {}, up_osds, osd_tree);
+        const aff_osds = this.get_affinity_osds(this.state.config.pools[pool_id], up_osds, osd_tree);
         const pg_items = {};
         this.reset_rng();
         new_pgs.map((osd_set, i) =>
@@ -1013,14 +1005,14 @@ class Mon
                 });
             }
         }
-        save_to.items = save_to.items || {};
+        this.state.config.pgs.items = this.state.config.pgs.items || {};
         if (!new_pgs.length)
         {
-            delete save_to.items[pool_id];
+            delete this.state.config.pgs.items[pool_id];
         }
         else
         {
-            save_to.items[pool_id] = pg_items;
+            this.state.config.pgs.items[pool_id] = pg_items;
         }
     }
 
@@ -1031,32 +1023,6 @@ class Mon
         pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
         pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
         pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
-        if (pool_cfg.failure_domain instanceof Object)
-        {
-            for (const key in pool_cfg.failure_domain)
-            {
-                const cnt = parseInt(pool_cfg.failure_domain[key]);
-                if (!cnt || cnt <= 0)
-                {
-                    if (warn)
-                        console.log('Pool '+pool_id+' specifies invalid item count for failure domain \"'+key+'\"');
-                    return false;
-                }
-                if (key !== 'host' && key != 'osd' && !(key in this.config.placement_levels||{}))
-                {
-                    if (warn)
-                        console.log('Pool '+pool_id+' uses invalid failure domain \"'+key+'\"');
-                    return false;
-                }
-            }
-        }
-        else if (pool_cfg.failure_domain !== 'host' && pool_cfg.failure_domain != 'osd' &&
-            !(pool_cfg.failure_domain in this.config.placement_levels||{}))
-        {
-            if (warn)
-                console.log('Pool '+pool_id+' uses invalid failure domain \"'+pool_cfg.failure_domain+'\"');
-            return false;
-        }
         pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
         if (!/^[1-9]\d*$/.exec(''+pool_id))
         {
@@ -1142,23 +1108,27 @@ class Mon
     filter_osds_by_tags(orig_tree, flat_tree, tags)
     {
         if (!tags)
-            return 1;
+        {
+            return;
+        }
         for (const tag of (tags instanceof Array ? tags : [ tags ]))
         {
-            for (const item in flat_tree)
+            for (const host in flat_tree)
             {
-                if (flat_tree[item] instanceof Object)
+                let found = 0;
+                for (const osd in flat_tree[host])
                 {
-                    if (!filter_osds_by_tags(orig_tree, flat_tree[item], tags))
-                        delete flat_tree[item];
+                    if (!orig_tree[osd].tags || !orig_tree[osd].tags[tag])
+                        delete flat_tree[host][osd];
+                    else
+                        found++;
+                }
+                if (!found)
+                {
+                    delete flat_tree[host];
                 }
-                else if (!orig_tree[item].tags || !orig_tree[item].tags[tag])
-                    delete flat_tree[item];
             }
         }
-        for (const item in flat_tree)
-            return 1;
-        return 0;
     }
 
     get_affinity_osds(pool_cfg, up_osds, osd_tree)
@@ -1186,7 +1156,6 @@ class Mon
         if (this.state.config.pgs.hash != tree_hash)
         {
             // Something has changed
-            const new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
             const etcd_request = { compare: [], success: [] };
             for (const pool_id in (this.state.config.pgs||{}).items||{})
             {
@@ -1207,7 +1176,7 @@ class Mon
                     etcd_request.success.push({ requestDeleteRange: {
                         key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
                     } });
-                    this.save_new_pgs_txn(new_config_pgs, etcd_request, pool_id, up_osds, osd_tree, prev_pgs, [], []);
+                    this.save_new_pgs_txn(etcd_request, pool_id, up_osds, osd_tree, prev_pgs, [], []);
                 }
             }
             for (const pool_id in this.state.config.pools)
@@ -1217,11 +1186,9 @@ class Mon
                 {
                     continue;
                 }
-                let pool_tree = osd_tree[pool_cfg.root_node || ''] || {};
-                const failure_domains = pool_cfg.failure_domain instanceof Object
-                    ? [ ...Object.keys(pool_cfg.failure_domain), 'osd' ]
-                    : [ pool_cfg.failure_domain, 'osd' ];
-                pool_tree = LPOptimizer.extract_tree_levels(pool_tree, failure_domains, levels);
+                let pool_tree = osd_tree[pool_cfg.root_node || ''];
+                pool_tree = pool_tree ? pool_tree.children : [];
+                pool_tree = LPOptimizer.flatten_tree(pool_tree, levels, pool_cfg.failure_domain, 'osd');
                 this.filter_osds_by_tags(osd_tree, pool_tree, pool_cfg.osd_tags);
                 // These are for the purpose of building history.osd_sets
                 const real_prev_pgs = [];
@@ -1248,9 +1215,6 @@ class Mon
                     pg_count: pool_cfg.pg_count,
                     pg_size: pool_cfg.pg_size,
                     pg_minsize: pool_cfg.pg_minsize,
-                    hier_sizes: pool_cfg.failure_domain instanceof Object
-                        ? [ ...Object.values(pool_cfg.failure_domain), 1 ]
-                        : null,
                     max_combinations: pool_cfg.max_osd_combinations,
                     ordered: pool_cfg.scheme != 'replicated',
                 };
@@ -1266,7 +1230,7 @@ class Mon
                             return;
                         }
                         const new_pg_history = [];
-                        PGUtil.scale_pg_count(prev_pgs, real_prev_pgs, pg_history, new_pg_history, pool_cfg.pg_count);
+                        PGUtil.scale_pg_count(prev_pgs, pg_history, new_pg_history, pool_cfg.pg_count);
                         pg_history = new_pg_history;
                     }
                     for (const pg of prev_pgs)
@@ -1306,7 +1270,7 @@ class Mon
                     } });
                 }
                 LPOptimizer.print_change_stats(optimize_result);
-                const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length); // FIXME requires hier support too
+                const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length);
                 this.state.pool.stats[pool_id] = {
                     used_raw_tb: (this.state.pool.stats[pool_id]||{}).used_raw_tb || 0,
                     total_raw_tb: optimize_result.space,
@@ -1319,15 +1283,14 @@ class Mon
                     key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
                     value: b64(JSON.stringify(this.state.pool.stats[pool_id])),
                 } });
-                this.save_new_pgs_txn(new_config_pgs, etcd_request, pool_id, up_osds, osd_tree, real_prev_pgs, optimize_result.int_pgs, pg_history);
+                this.save_new_pgs_txn(etcd_request, pool_id, up_osds, osd_tree, real_prev_pgs, optimize_result.int_pgs, pg_history);
             }
-            new_config_pgs.hash = tree_hash;
-            await this.save_pg_config(new_config_pgs, etcd_request);
+            this.state.config.pgs.hash = tree_hash;
+            await this.save_pg_config(etcd_request);
         }
         else
         {
             // Nothing changed, but we still want to recheck the distribution of primaries
-            let new_config_pgs;
             let changed = false;
             for (const pool_id in this.state.config.pools)
             {
@@ -1347,35 +1310,31 @@ class Mon
                         const new_primary = this.pick_primary(pool_id, pg_cfg.osd_set, up_osds, aff_osds);
                         if (pg_cfg.primary != new_primary)
                         {
-                            if (!new_config_pgs)
-                            {
-                                new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
-                            }
                             console.log(
                                 `Moving pool ${pool_id} (${pool_cfg.name || 'unnamed'}) PG ${pg_num}`+
                                 ` primary OSD from ${pg_cfg.primary} to ${new_primary}`
                             );
                             changed = true;
-                            new_config_pgs.items[pool_id][pg_num].primary = new_primary;
+                            pg_cfg.primary = new_primary;
                         }
                     }
                 }
             }
             if (changed)
             {
-                await this.save_pg_config(new_config_pgs);
+                await this.save_pg_config();
             }
         }
     }
 
-    async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
+    async save_pg_config(etcd_request = { compare: [], success: [] })
     {
         etcd_request.compare.push(
             { key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
             { key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
         );
         etcd_request.success.push(
-            { requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } },
+            { requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(this.state.config.pgs)) } },
         );
         const res = await this.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0);
         if (!res.succeeded)
@@ -1854,7 +1813,6 @@ function POST(url, body, timeout)
             clearTimeout(timer_id);
             let res_body = '';
             res.setEncoding('utf8');
-            res.on('error', (error) => ok({ error }));
             res.on('data', chunk => { res_body += chunk; });
             res.on('end', () =>
             {
@@ -1874,8 +1832,6 @@ function POST(url, body, timeout)
                 }
             });
         });
-        req.on('error', (error) => ok({ error }));
-        req.on('close', () => ok({ error: new Error('Connection closed prematurely') }));
         req.write(body_text);
         req.end();
     });

mon/test-optimize-undersized.js

@@ -36,7 +36,7 @@ const crush_tree = [
     ] },
 ];
 
-const osd_tree = LPOptimizer.extract_tree_levels({ level: -Infinity, children: crush_tree }, [ 1, 3 ], {});
+const osd_tree = LPOptimizer.flatten_tree(crush_tree, {}, 1, 3);
 console.log(osd_tree);
 
 async function run()
@@ -47,32 +47,32 @@ async function run()
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 0);
     console.log('\nAdding 1st failure domain:');
-    cur_tree['l1_1'] = osd_tree['l1_1'];
+    cur_tree['dom1'] = osd_tree['dom1'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 12 && res.total_space == 12);
     console.log('\nAdding 2nd failure domain:');
-    cur_tree['l1_2'] = osd_tree['l1_2'];
+    cur_tree['dom2'] = osd_tree['dom2'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 24 && res.total_space == 24);
     console.log('\nAdding 3rd failure domain:');
-    cur_tree['l1_3'] = osd_tree['l1_3'];
+    cur_tree['dom3'] = osd_tree['dom3'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 36 && res.total_space == 36);
     console.log('\nRemoving 3rd failure domain:');
-    delete cur_tree['l1_3'];
+    delete cur_tree['dom3'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 24 && res.total_space == 24);
     console.log('\nRemoving 2nd failure domain:');
-    delete cur_tree['l1_2'];
+    delete cur_tree['dom2'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 12 && res.total_space == 12);
     console.log('\nRemoving 1st failure domain:');
-    delete cur_tree['l1_1'];
+    delete cur_tree['dom1'];
     res = await LPOptimizer.optimize_change({ prev_pgs: res.int_pgs, osd_tree: cur_tree, pg_size: 3 });
     LPOptimizer.print_change_stats(res, false);
     assert(res.space == 0);

mon/test-optimize.js

@@ -108,11 +108,7 @@ async function run()
     LPOptimizer.print_change_stats(res, false);
 
     console.log('\n256 PGs, size=3, failure domain=rack');
-    res = await LPOptimizer.optimize_initial({
-        osd_tree: LPOptimizer.extract_tree_levels({ level: -Infinity, children: crush_tree }, [ 1, 3 ], {}),
-        pg_size: 3,
-        pg_count: 256,
-    });
+    res = await LPOptimizer.optimize_initial({ osd_tree: LPOptimizer.flatten_tree(crush_tree, {}, 1, 3), pg_size: 3, pg_count: 256 });
     LPOptimizer.print_change_stats(res, false);
 }
 

mon/test-random-hier.js

@@ -1,56 +0,0 @@
-// Copyright (c) Vitaliy Filippov, 2019+
-// License: VNPL-1.1 (see README.md for details)
-
-const LPOptimizer = require('./lp-optimizer.js');
-
-const osd_tree = {
-    100: { 110: { 111: 1, 112: 1 }, 120: { 121: 1, 122: 1 } },
-    200: { 210: { 211: 1, 212: 1 }, 220: { 221: 1, 222: 1 } },
-    300: { 310: { 311: 1, 312: 1 }, 320: { 321: 1, 322: 1 } },
-    400: { 410: { 411: 1, 412: 1 }, 420: { 421: 1, 422: 1 } },
-    500: { 510: { 511: 1, 512: 1 }, 520: { 521: 1, 522: 1 } },
-};
-
-const osd_tree2 = {
-    100: { 111: 1, 112: 1, 121: 1, 122: 1 },
-    200: { 211: 1, 212: 1, 221: 1, 222: 1 },
-    300: { 311: 1, 312: 1, 321: 1, 322: 1 },
-    400: { 411: 1, 412: 1, 421: 1, 422: 1 },
-    500: { 511: 1, 512: 1, 521: 1, 522: 1 },
-};
-
-const osd_tree3 = {
-    100: { 111: 1, 112: 1, 121: 1, 122: 1 },
-    200: { 211: 1, 212: 1, 221: 1, 222: 1 },
-    300: { 311: 1, 312: 1, 321: 1, 322: 1 },
-    400: { 411: 1, 412: 1, 421: 1, 422: 1 },
-    500: { 511: 1 },
-};
-
-async function run()
-{
-    let r;
-    console.log(r = LPOptimizer.random_hier_combinations(osd_tree, [ 3, 2, 1 ], 10000, false, true));
-    console.log(r = LPOptimizer.random_hier_combinations(osd_tree2, [ 3, 2 ], 0, false, true));
-    // Will contain 'Z':
-    console.log(r = LPOptimizer.random_combinations(osd_tree2, 6, 0, true));
-    console.log(r = LPOptimizer.extract_tree_levels(
-        { level: 'dc', children: [
-            { level: 'rack', children: [
-                { level: 'host', children: [
-                    { level: 'osd', id: 'OSD5', size: 10 },
-                ] },
-            ] },
-            { level: 'osd', id: 'OSD10', size: 10 },
-        ] },
-        [ 'rack', 'osd' ],
-        { dc: 1, rack: 2, host: 3, osd: 4 }
-    ));
-    if (JSON.stringify(r) != '{"rack1":{"OSD5":10},"rack2":{"OSD10":10}}')
-        throw new Error('extract_tree_levels failed');
-    // should not contain Z:
-    console.log(r = LPOptimizer.random_hier_combinations(osd_tree3, [ 3, 2 ], 0, false, true));
-    console.log('OK');
-}
-
-run().catch(console.error);

mon/vitastor-mon.service

@@ -15,4 +15,4 @@ StartLimitInterval=0
 RestartSec=10
 
 [Install]
-WantedBy=multi-user.target
+WantedBy=vitastor.target

patches/VitastorPlugin.pm

@@ -16,11 +16,6 @@ use PVE::Tools qw(run_command);
 
 use base qw(PVE::Storage::Plugin);
 
-if (@PVE::Storage::Plugin::SHARED_STORAGE)
-{
-    push @PVE::Storage::Plugin::SHARED_STORAGE, 'vitastor';
-}
-
 sub api
 {
     # Trick it :)
@@ -138,11 +133,9 @@ sub properties
 sub options
 {
     return {
-        shared => { optional => 1 },
-        content => { optional => 1 },
         nodes => { optional => 1 },
         disable => { optional => 1 },
-        vitastor_etcd_address => { optional => 1 },
+        vitastor_etcd_address => { optional => 1},
         vitastor_etcd_prefix => { optional => 1 },
         vitastor_config_path => { optional => 1 },
         vitastor_prefix => { optional => 1 },

patches/cinder-vitastor.py

@@ -50,7 +50,7 @@ from cinder.volume import configuration
 from cinder.volume import driver
 from cinder.volume import volume_utils
 
-VERSION = '0.8.9'
+VERSION = '0.8.3'
 
 LOG = logging.getLogger(__name__)
 

patches/pve-qemu-7.2-vitastor.patch

@@ -1,169 +0,0 @@
-Index: pve-qemu-kvm-7.2.0/block/meson.build
-===================================================================
---- pve-qemu-kvm-7.2.0.orig/block/meson.build
-+++ pve-qemu-kvm-7.2.0/block/meson.build
-@@ -113,6 +113,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-Index: pve-qemu-kvm-7.2.0/meson.build
-===================================================================
---- pve-qemu-kvm-7.2.0.orig/meson.build
-+++ pve-qemu-kvm-7.2.0/meson.build
-@@ -1026,6 +1026,26 @@ if not get_option('rbd').auto() or have_
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1865,6 +1885,7 @@ config_host_data.set('CONFIG_NUMA', numa
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
-@@ -3957,6 +3978,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-Index: pve-qemu-kvm-7.2.0/meson_options.txt
-===================================================================
---- pve-qemu-kvm-7.2.0.orig/meson_options.txt
-+++ pve-qemu-kvm-7.2.0/meson_options.txt
-@@ -169,6 +169,8 @@ option('lzo', type : 'feature', value :
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-Index: pve-qemu-kvm-7.2.0/qapi/block-core.json
-===================================================================
---- pve-qemu-kvm-7.2.0.orig/qapi/block-core.json
-+++ pve-qemu-kvm-7.2.0/qapi/block-core.json
-@@ -3213,7 +3213,7 @@
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
-             'pbs',
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4223,6 +4223,28 @@
-             '*server': ['InetSocketAddressBase'] } }
- 
- ##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
-+##
- # @ReplicationMode:
- #
- # An enumeration of replication modes.
-@@ -4671,6 +4693,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -5072,6 +5095,17 @@
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
- ##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
-+##
- # @BlockdevVmdkSubformat:
- #
- # Subformat options for VMDK images
-@@ -5269,6 +5303,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-Index: pve-qemu-kvm-7.2.0/scripts/ci/org.centos/stream/8/x86_64/configure
-===================================================================
---- pve-qemu-kvm-7.2.0.orig/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ pve-qemu-kvm-7.2.0/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -31,7 +31,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -179,6 +179,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \

patches/qemu-6.2-vitastor.patch

@@ -1,169 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index deb73ca389..e269f599a1 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -78,6 +78,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 96de1a6ef9..2e3994777d 100644
---- a/meson.build
-+++ b/meson.build
-@@ -838,6 +838,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1455,6 +1475,7 @@ config_host_data.set('CONFIG_LINUX_AIO', libaio.found())
- config_host_data.set('CONFIG_LINUX_IO_URING', linux_io_uring.found())
- config_host_data.set('CONFIG_LIBPMEM', libpmem.found())
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
- config_host_data.set('CONFIG_SECCOMP', seccomp.found())
-@@ -3412,6 +3433,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'xfsctl support':    config_host.has_key('CONFIG_XFS')}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
-diff --git a/meson_options.txt b/meson_options.txt
-index e392323732..5b56007475 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -121,6 +121,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('gtk', type : 'feature', value : 'auto',
-        description: 'GTK+ user interface')
- option('sdl', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index 1d3dd9cb48..88453405e5 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -2930,7 +2930,7 @@
-             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
-             'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor', 'vmdk', 'vpc', 'vvfat' ] }
- 
- ##
- # @BlockdevOptionsFile:
-@@ -3864,6 +3864,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4259,6 +4281,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'vmdk':       'BlockdevOptionsGenericCOWFormat',
-       'vpc':        'BlockdevOptionsGenericFormat',
-       'vvfat':      'BlockdevOptionsVVFAT'
-@@ -4647,6 +4670,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -4846,6 +4880,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 7a17ff4218..cdddbf32aa 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -69,6 +69,7 @@ meson_options_help() {
-   printf "%s\n" '  oss             OSS sound support'
-   printf "%s\n" '  pa              PulseAudio sound support'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  sdl             SDL user interface'
-   printf "%s\n" '  sdl-image       SDL Image support for icons'
-   printf "%s\n" '  seccomp         seccomp support'
-@@ -210,6 +211,8 @@ _meson_option_parse() {
-     --disable-pa) printf "%s" -Dpa=disabled ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-sdl) printf "%s" -Dsdl=enabled ;;
-     --disable-sdl) printf "%s" -Dsdl=disabled ;;
-     --enable-sdl-image) printf "%s" -Dsdl_image=enabled ;;

patches/qemu-7.0-vitastor.patch

@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 0b2a60c99b..d923713804 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -98,6 +98,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 861de93c4f..272f72af11 100644
---- a/meson.build
-+++ b/meson.build
-@@ -884,6 +884,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1546,6 +1566,7 @@ config_host_data.set('CONFIG_LIBPMEM', libpmem.found())
- config_host_data.set('CONFIG_NUMA', numa.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
- config_host_data.set('CONFIG_SECCOMP', seccomp.found())
-@@ -3709,6 +3730,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index 52b11cead4..d8d0868174 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -149,6 +149,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('gtk', type : 'feature', value : 'auto',
-        description: 'GTK+ user interface')
- option('sdl', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index beeb91952a..1c98dc0e12 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -2929,7 +2929,7 @@
-             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
-             'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor', 'vmdk', 'vpc', 'vvfat' ] }
- 
- ##
- # @BlockdevOptionsFile:
-@@ -3863,6 +3863,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4277,6 +4299,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'vmdk':       'BlockdevOptionsGenericCOWFormat',
-       'vpc':        'BlockdevOptionsGenericFormat',
-       'vvfat':      'BlockdevOptionsVVFAT'
-@@ -4665,6 +4688,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -4864,6 +4898,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index 9850dd4444..72b1287520 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -31,7 +31,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -181,6 +181,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 1e26f4571e..370898d48c 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -98,6 +98,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  sdl             SDL user interface'
-   printf "%s\n" '  sdl-image       SDL Image support for icons'
-@@ -289,6 +290,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-replication) printf "%s" -Dreplication=enabled ;;
-     --disable-replication) printf "%s" -Dreplication=disabled ;;
-     --enable-rng-none) printf "%s" -Drng_none=true ;;

patches/qemu-7.1-vitastor.patch

@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 60bc305597..89a042216f 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -98,6 +98,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 20fddbd707..600db4e2fb 100644
---- a/meson.build
-+++ b/meson.build
-@@ -967,6 +967,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1799,6 +1819,7 @@ config_host_data.set('CONFIG_NUMA', numa.found())
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
-@@ -3954,6 +3975,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index e58e158396..9747b38fd0 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -167,6 +167,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index 2173e7734a..5a4900b322 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -2955,7 +2955,7 @@
-             'luks', 'nbd', 'nfs', 'null-aio', 'null-co', 'nvme', 'parallels',
-             'preallocate', 'qcow', 'qcow2', 'qed', 'quorum', 'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx', 'vmdk', 'vpc', 'vvfat' ] }
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor', 'vmdk', 'vpc', 'vvfat' ] }
- 
- ##
- # @BlockdevOptionsFile:
-@@ -3883,6 +3883,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4327,6 +4349,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'vmdk':       'BlockdevOptionsGenericCOWFormat',
-       'vpc':        'BlockdevOptionsGenericFormat',
-       'vvfat':      'BlockdevOptionsVVFAT'
-@@ -4717,6 +4740,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -4915,6 +4949,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index a7f92aff90..53dc55be2e 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -31,7 +31,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -179,6 +179,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 359b04e0e6..f5b85ba78c 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -135,6 +135,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  sdl             SDL user interface'
-@@ -370,6 +371,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-replication) printf "%s" -Dreplication=enabled ;;

patches/qemu-7.2-vitastor.patch

@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index b7c68b83a3..95d8a6f15d 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -100,6 +100,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index 5c6b5a1c75..f31f73612e 100644
---- a/meson.build
-+++ b/meson.build
-@@ -1026,6 +1026,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1861,6 +1881,7 @@ config_host_data.set('CONFIG_NUMA', numa.found())
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
-@@ -3945,6 +3966,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index 4b749ca549..6b37bd6b77 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -169,6 +169,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index 95ac4fa634..7a240827e4 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -2959,7 +2959,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -3957,6 +3957,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4405,6 +4427,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -4804,6 +4827,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5002,6 +5036,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index a7f92aff90..53dc55be2e 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -31,7 +31,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -179,6 +179,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index aa6e30ea91..c45d21c40f 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -135,6 +135,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  sdl             SDL user interface'
-@@ -376,6 +377,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-replication) printf "%s" -Dreplication=enabled ;;

patches/qemu-8.0-vitastor.patch

@@ -1,190 +0,0 @@
-diff --git a/block/meson.build b/block/meson.build
-index 382bec0e7d..af6207dbce 100644
---- a/block/meson.build
-+++ b/block/meson.build
-@@ -101,6 +101,7 @@ foreach m : [
-   [libnfs, 'nfs', files('nfs.c')],
-   [libssh, 'ssh', files('ssh.c')],
-   [rbd, 'rbd', files('rbd.c')],
-+  [vitastor, 'vitastor', files('vitastor.c')],
- ]
-   if m[0].found()
-     module_ss = ss.source_set()
-diff --git a/meson.build b/meson.build
-index c44d05a13f..ebedb42843 100644
---- a/meson.build
-+++ b/meson.build
-@@ -1028,6 +1028,26 @@ if not get_option('rbd').auto() or have_block
-   endif
- endif
- 
-+vitastor = not_found
-+if not get_option('vitastor').auto() or have_block
-+  libvitastor_client = cc.find_library('vitastor_client', has_headers: ['vitastor_c.h'],
-+    required: get_option('vitastor'), kwargs: static_kwargs)
-+  if libvitastor_client.found()
-+    if cc.links('''
-+      #include <vitastor_c.h>
-+      int main(void) {
-+        vitastor_c_create_qemu(0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
-+        return 0;
-+      }''', dependencies: libvitastor_client)
-+      vitastor = declare_dependency(dependencies: libvitastor_client)
-+    elif get_option('vitastor').enabled()
-+      error('could not link libvitastor_client')
-+    else
-+      warning('could not link libvitastor_client, disabling')
-+    endif
-+  endif
-+endif
-+
- glusterfs = not_found
- glusterfs_ftruncate_has_stat = false
- glusterfs_iocb_has_stat = false
-@@ -1878,6 +1898,7 @@ endif
- config_host_data.set('CONFIG_OPENGL', opengl.found())
- config_host_data.set('CONFIG_PROFILER', get_option('profiler'))
- config_host_data.set('CONFIG_RBD', rbd.found())
-+config_host_data.set('CONFIG_VITASTOR', vitastor.found())
- config_host_data.set('CONFIG_RDMA', rdma.found())
- config_host_data.set('CONFIG_SDL', sdl.found())
- config_host_data.set('CONFIG_SDL_IMAGE', sdl_image.found())
-@@ -4002,6 +4023,7 @@ if spice_protocol.found()
-   summary_info += {'  spice server support': spice}
- endif
- summary_info += {'rbd support':       rbd}
-+summary_info += {'vitastor support':  vitastor}
- summary_info += {'smartcard support': cacard}
- summary_info += {'U2F support':       u2f}
- summary_info += {'libusb':            libusb}
-diff --git a/meson_options.txt b/meson_options.txt
-index fc9447d267..c4ac55c283 100644
---- a/meson_options.txt
-+++ b/meson_options.txt
-@@ -173,6 +173,8 @@ option('lzo', type : 'feature', value : 'auto',
-        description: 'lzo compression support')
- option('rbd', type : 'feature', value : 'auto',
-        description: 'Ceph block device driver')
-+option('vitastor', type : 'feature', value : 'auto',
-+       description: 'Vitastor block device driver')
- option('opengl', type : 'feature', value : 'auto',
-        description: 'OpenGL support')
- option('rdma', type : 'feature', value : 'auto',
-diff --git a/qapi/block-core.json b/qapi/block-core.json
-index c05ad0c07e..f5eb701604 100644
---- a/qapi/block-core.json
-+++ b/qapi/block-core.json
-@@ -3054,7 +3054,7 @@
-             'parallels', 'preallocate', 'qcow', 'qcow2', 'qed', 'quorum',
-             'raw', 'rbd',
-             { 'name': 'replication', 'if': 'CONFIG_REPLICATION' },
--            'ssh', 'throttle', 'vdi', 'vhdx',
-+            'ssh', 'throttle', 'vdi', 'vhdx', 'vitastor',
-             { 'name': 'virtio-blk-vfio-pci', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-user', 'if': 'CONFIG_BLKIO' },
-             { 'name': 'virtio-blk-vhost-vdpa', 'if': 'CONFIG_BLKIO' },
-@@ -4073,6 +4073,28 @@
-             '*key-secret': 'str',
-             '*server': ['InetSocketAddressBase'] } }
- 
-+##
-+# @BlockdevOptionsVitastor:
-+#
-+# Driver specific block device options for vitastor
-+#
-+# @image:       Image name
-+# @inode:       Inode number
-+# @pool:        Pool ID
-+# @size:        Desired image size in bytes
-+# @config-path: Path to Vitastor configuration
-+# @etcd-host:   etcd connection address(es)
-+# @etcd-prefix: etcd key/value prefix
-+##
-+{ 'struct': 'BlockdevOptionsVitastor',
-+  'data': { '*inode': 'uint64',
-+            '*pool': 'uint64',
-+            '*size': 'uint64',
-+            '*image': 'str',
-+            '*config-path': 'str',
-+            '*etcd-host': 'str',
-+            '*etcd-prefix': 'str' } }
-+
- ##
- # @ReplicationMode:
- #
-@@ -4521,6 +4543,7 @@
-       'throttle':   'BlockdevOptionsThrottle',
-       'vdi':        'BlockdevOptionsGenericFormat',
-       'vhdx':       'BlockdevOptionsGenericFormat',
-+      'vitastor':   'BlockdevOptionsVitastor',
-       'virtio-blk-vfio-pci':
-                     { 'type': 'BlockdevOptionsVirtioBlkVfioPci',
-                       'if': 'CONFIG_BLKIO' },
-@@ -4920,6 +4943,17 @@
-             '*cluster-size' :   'size',
-             '*encrypt' :        'RbdEncryptionCreateOptions' } }
- 
-+##
-+# @BlockdevCreateOptionsVitastor:
-+#
-+# Driver specific image creation options for Vitastor.
-+#
-+# @size: Size of the virtual disk in bytes
-+##
-+{ 'struct': 'BlockdevCreateOptionsVitastor',
-+  'data': { 'location':         'BlockdevOptionsVitastor',
-+            'size':             'size' } }
-+
- ##
- # @BlockdevVmdkSubformat:
- #
-@@ -5118,6 +5152,7 @@
-       'ssh':            'BlockdevCreateOptionsSsh',
-       'vdi':            'BlockdevCreateOptionsVdi',
-       'vhdx':           'BlockdevCreateOptionsVhdx',
-+      'vitastor':       'BlockdevCreateOptionsVitastor',
-       'vmdk':           'BlockdevCreateOptionsVmdk',
-       'vpc':            'BlockdevCreateOptionsVpc'
-   } }
-diff --git a/scripts/ci/org.centos/stream/8/x86_64/configure b/scripts/ci/org.centos/stream/8/x86_64/configure
-index 6e8983f39c..1b0b9fcf3e 100755
---- a/scripts/ci/org.centos/stream/8/x86_64/configure
-+++ b/scripts/ci/org.centos/stream/8/x86_64/configure
-@@ -32,7 +32,7 @@
- --with-git=meson \
- --with-git-submodules=update \
- --target-list="x86_64-softmmu" \
----block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
-+--block-drv-rw-whitelist="qcow2,raw,file,host_device,nbd,iscsi,rbd,vitastor,blkdebug,luks,null-co,nvme,copy-on-read,throttle,gluster" \
- --audio-drv-list="" \
- --block-drv-ro-whitelist="vmdk,vhdx,vpc,https,ssh" \
- --with-coroutine=ucontext \
-@@ -179,6 +179,7 @@
- --enable-opengl \
- --enable-pie \
- --enable-rbd \
-+--enable-vitastor \
- --enable-rdma \
- --enable-seccomp \
- --enable-snappy \
-diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh
-index 009fab1515..95914e6ebc 100644
---- a/scripts/meson-buildoptions.sh
-+++ b/scripts/meson-buildoptions.sh
-@@ -142,6 +142,7 @@ meson_options_help() {
-   printf "%s\n" '  qed             qed image format support'
-   printf "%s\n" '  qga-vss         build QGA VSS support (broken with MinGW)'
-   printf "%s\n" '  rbd             Ceph block device driver'
-+  printf "%s\n" '  vitastor        Vitastor block device driver'
-   printf "%s\n" '  rdma            Enable RDMA-based migration'
-   printf "%s\n" '  replication     replication support'
-   printf "%s\n" '  sdl             SDL user interface'
-@@ -388,6 +389,8 @@ _meson_option_parse() {
-     --disable-qom-cast-debug) printf "%s" -Dqom_cast_debug=false ;;
-     --enable-rbd) printf "%s" -Drbd=enabled ;;
-     --disable-rbd) printf "%s" -Drbd=disabled ;;
-+    --enable-vitastor) printf "%s" -Dvitastor=enabled ;;
-+    --disable-vitastor) printf "%s" -Dvitastor=disabled ;;
-     --enable-rdma) printf "%s" -Drdma=enabled ;;
-     --disable-rdma) printf "%s" -Drdma=disabled ;;
-     --enable-replication) printf "%s" -Dreplication=enabled ;;

rpm/build-tarball.sh

@@ -7,12 +7,13 @@ set -e
 VITASTOR=$(dirname $0)
 VITASTOR=$(realpath "$VITASTOR/..")
 
-EL=$(rpm --eval '%dist')
-if [ "$EL" = ".el8" ]; then
+if [ -d /opt/rh/gcc-toolset-9 ]; then
     # CentOS 8
+    EL=8
     . /opt/rh/gcc-toolset-9/enable
-elif [ "$EL" = ".el7" ]; then
+else
     # CentOS 7
+    EL=7
     . /opt/rh/devtoolset-9/enable
 fi
 cd ~/rpmbuild/SPECS
@@ -24,4 +25,4 @@ rm fio
 mv fio-copy fio
 FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'`
 perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec
-tar --transform 's#^#vitastor-0.8.9/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.9$(rpm --eval '%dist').tar.gz *
+tar --transform 's#^#vitastor-0.8.3/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.8.3$(rpm --eval '%dist').tar.gz *

rpm/qemu-kvm-7.0-el9.spec.patch

@@ -1,93 +0,0 @@
---- qemu-kvm.spec.orig	2023-02-28 08:04:06.000000000 +0000
-+++ qemu-kvm.spec	2023-04-27 22:29:18.094878829 +0000
-@@ -100,8 +100,6 @@
- %endif
- 
- %global target_list %{kvm_target}-softmmu
--%global block_drivers_rw_list qcow2,raw,file,host_device,nbd,iscsi,rbd,blkdebug,luks,null-co,nvme,copy-on-read,throttle,compress
--%global block_drivers_ro_list vdi,vmdk,vhdx,vpc,https
- %define qemudocdir %{_docdir}/%{name}
- %global firmwaredirs "%{_datadir}/qemu-firmware:%{_datadir}/ipxe/qemu:%{_datadir}/seavgabios:%{_datadir}/seabios"
- 
-@@ -129,6 +127,7 @@ Requires: %{name}-device-usb-host = %{ep
- Requires: %{name}-device-usb-redirect = %{epoch}:%{version}-%{release}   \
- %endif                                                           \
- Requires: %{name}-block-rbd = %{epoch}:%{version}-%{release}     \
-+Requires: %{name}-block-vitastor = %{epoch}:%{version}-%{release}\
- Requires: %{name}-audio-pa = %{epoch}:%{version}-%{release}
- 
- # Since SPICE is removed from RHEL-9, the following Obsoletes:
-@@ -151,7 +150,7 @@ Obsoletes: %{name}-block-ssh <= %{epoch}
- Summary: QEMU is a machine emulator and virtualizer
- Name: qemu-kvm
- Version: 7.0.0
--Release: 13%{?rcrel}%{?dist}%{?cc_suffix}.2
-+Release: 13.vitastor%{?rcrel}%{?dist}%{?cc_suffix}
- # Epoch because we pushed a qemu-1.0 package. AIUI this can't ever be dropped
- # Epoch 15 used for RHEL 8
- # Epoch 17 used for RHEL 9 (due to release versioning offset in RHEL 8.5)
-@@ -174,6 +173,7 @@ Source28: 95-kvm-memlock.conf
- Source30: kvm-s390x.conf
- Source31: kvm-x86.conf
- Source36: README.tests
-+Source37: qemu-vitastor.c
- 
- 
- Patch0004: 0004-Initial-redhat-build.patch
-@@ -498,6 +498,7 @@ Patch171: kvm-i386-do-kvm_put_msr_featur
- Patch172: kvm-target-i386-kvm-fix-kvmclock_current_nsec-Assertion-.patch
- # For bz#2168221 - while live-migrating many instances concurrently, libvirt sometimes return internal error: migration was active, but no RAM info was set [rhel-9.1.0.z]
- Patch173: kvm-migration-Read-state-once.patch
-+Patch174: qemu-7.0-vitastor.patch
- 
- # Source-git patches
- 
-@@ -531,6 +532,7 @@ BuildRequires: libcurl-devel
- %if %{have_block_rbd}
- BuildRequires: librbd-devel
- %endif
-+BuildRequires: vitastor-client-devel
- # We need both because the 'stap' binary is probed for by configure
- BuildRequires: systemtap
- BuildRequires: systemtap-sdt-devel
-@@ -718,6 +720,14 @@ using the rbd protocol.
- %endif
- 
- 
-+%package  block-vitastor
-+Summary: QEMU Vitastor block driver
-+Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-+
-+%description block-vitastor
-+This package provides the additional Vitastor block driver for QEMU.
-+
-+
- %package  audio-pa
- Summary: QEMU PulseAudio audio driver
- Requires: %{name}-common%{?_isa} = %{epoch}:%{version}-%{release}
-@@ -811,6 +821,7 @@ This package provides usbredir support.
- %prep
- %setup -q -n qemu-%{version}%{?rcstr}
- %autopatch -p1
-+cp %{SOURCE37} ./block/vitastor.c
- 
- %global qemu_kvm_build qemu_kvm_build
- mkdir -p %{qemu_kvm_build}
-@@ -1032,6 +1043,7 @@ run_configure \
- %if %{have_block_rbd}
-   --enable-rbd \
- %endif
-+  --enable-vitastor \
- %if %{have_librdma}
-   --enable-rdma \
- %endif
-@@ -1511,6 +1523,9 @@ useradd -r -u 107 -g qemu -G kvm -d / -s
- %files block-rbd
- %{_libdir}/%{name}/block-rbd.so
- %endif
-+%files block-vitastor
-+%{_libdir}/%{name}/block-vitastor.so
-+
- %files audio-pa
- %{_libdir}/%{name}/audio-pa.so
- 

rpm/vitastor-el7.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.9.el7.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el7.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el7.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el7.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.9
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.9.el7.tar.gz
+Source0:        vitastor-0.8.3.el7.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -35,7 +35,6 @@ Summary:        Vitastor - OSD
 Requires:       libJerasure2
 Requires:       libisa-l
 Requires:       liburing >= 0.6
-Requires:       liburing < 2
 Requires:       vitastor-client = %{version}-%{release}
 Requires:       util-linux
 Requires:       parted
@@ -60,7 +59,6 @@ scheduling cluster-level operations.
 %package -n vitastor-client
 Summary:        Vitastor - client
 Requires:       liburing >= 0.6
-Requires:       liburing < 2
 
 
 %description -n vitastor-client

rpm/vitastor-el8.Dockerfile

@@ -35,7 +35,7 @@ ADD . /root/vitastor
 RUN set -e; \
     cd /root/vitastor/rpm; \
     sh build-tarball.sh; \
-    cp /root/vitastor-0.8.9.el8.tar.gz ~/rpmbuild/SOURCES; \
+    cp /root/vitastor-0.8.3.el8.tar.gz ~/rpmbuild/SOURCES; \
     cp vitastor-el8.spec ~/rpmbuild/SPECS/vitastor.spec; \
     cd ~/rpmbuild/SPECS/; \
     rpmbuild -ba vitastor.spec; \

rpm/vitastor-el8.spec

@@ -1,11 +1,11 @@
 Name:           vitastor
-Version:        0.8.9
+Version:        0.8.3
 Release:        1%{?dist}
 Summary:        Vitastor, a fast software-defined clustered block storage
 
 License:        Vitastor Network Public License 1.1
 URL:            https://vitastor.io/
-Source0:        vitastor-0.8.9.el8.tar.gz
+Source0:        vitastor-0.8.3.el8.tar.gz
 
 BuildRequires:  liburing-devel >= 0.6
 BuildRequires:  gperftools-devel
@@ -34,7 +34,6 @@ Summary:        Vitastor - OSD
 Requires:       libJerasure2
 Requires:       libisa-l
 Requires:       liburing >= 0.6
-Requires:       liburing < 2
 Requires:       vitastor-client = %{version}-%{release}
 Requires:       util-linux
 Requires:       parted
@@ -58,7 +57,6 @@ scheduling cluster-level operations.
 %package -n vitastor-client
 Summary:        Vitastor - client
 Requires:       liburing >= 0.6
-Requires:       liburing < 2
 
 
 %description -n vitastor-client

rpm/vitastor-el9.Dockerfile

@@ -1,28 +0,0 @@
-# Build packages for AlmaLinux 9 inside a container
-# cd ..; podman build -t vitastor-el9 -v `pwd`/packages:/root/packages -f rpm/vitastor-el9.Dockerfile .
-
-FROM almalinux:9
-
-WORKDIR /root
-
-RUN sed -i 's/enabled=0/enabled=1/' /etc/yum.repos.d/*.repo
-RUN dnf -y install epel-release dnf-plugins-core
-RUN dnf -y install https://vitastor.io/rpms/centos/9/vitastor-release-1.0-1.el9.noarch.rpm
-RUN dnf -y install gcc-c++ gperftools-devel fio nodejs rpm-build jerasure-devel libisa-l-devel gf-complete-devel rdma-core-devel libarchive liburing-devel cmake
-RUN dnf download --source fio
-RUN rpm --nomd5 -i fio*.src.rpm
-RUN cd ~/rpmbuild/SPECS && dnf builddep -y --spec fio.spec
-
-ADD . /root/vitastor
-
-RUN set -e; \
-    cd /root/vitastor/rpm; \
-    sh build-tarball.sh; \
-    cp /root/vitastor-0.8.9.el9.tar.gz ~/rpmbuild/SOURCES; \
-    cp vitastor-el9.spec ~/rpmbuild/SPECS/vitastor.spec; \
-    cd ~/rpmbuild/SPECS/; \
-    rpmbuild -ba vitastor.spec; \
-    mkdir -p /root/packages/vitastor-el9; \
-    rm -rf /root/packages/vitastor-el9/*; \
-    cp ~/rpmbuild/RPMS/*/vitastor* /root/packages/vitastor-el9/; \
-    cp ~/rpmbuild/SRPMS/vitastor* /root/packages/vitastor-el9/

rpm/vitastor-el9.spec

@@ -1,158 +0,0 @@
-Name:           vitastor
-Version:        0.8.9
-Release:        1%{?dist}
-Summary:        Vitastor, a fast software-defined clustered block storage
-
-License:        Vitastor Network Public License 1.1
-URL:            https://vitastor.io/
-Source0:        vitastor-0.8.9.el9.tar.gz
-
-BuildRequires:  liburing-devel >= 0.6
-BuildRequires:  gperftools-devel
-BuildRequires:  gcc-c++
-BuildRequires:  nodejs >= 10
-BuildRequires:  jerasure-devel
-BuildRequires:  libisa-l-devel
-BuildRequires:  gf-complete-devel
-BuildRequires:  rdma-core-devel
-BuildRequires:  cmake
-Requires:       vitastor-osd = %{version}-%{release}
-Requires:       vitastor-mon = %{version}-%{release}
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       vitastor-client-devel = %{version}-%{release}
-Requires:       vitastor-fio = %{version}-%{release}
-
-%description
-Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
-architecturally similar to Ceph which means strong consistency, primary-replication,
-symmetric clustering and automatic data distribution over any number of drives of any
-size with configurable redundancy (replication or erasure codes/XOR).
-
-
-%package -n vitastor-osd
-Summary:        Vitastor - OSD
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       util-linux
-Requires:       parted
-
-
-%description -n vitastor-osd
-Vitastor object storage daemon, i.e. server program that stores data.
-
-
-%package -n vitastor-mon
-Summary:        Vitastor - monitor
-Requires:       nodejs >= 10
-Requires:       lpsolve
-
-
-%description -n vitastor-mon
-Vitastor monitor, i.e. server program responsible for watching cluster state and
-scheduling cluster-level operations.
-
-
-%package -n vitastor-client
-Summary:        Vitastor - client
-
-
-%description -n vitastor-client
-Vitastor client library and command-line interface.
-
-
-%package -n vitastor-client-devel
-Summary:        Vitastor - development files
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-
-
-%description -n vitastor-client-devel
-Vitastor library headers for development.
-
-
-%package -n vitastor-fio
-Summary:        Vitastor - fio drivers
-Group:          Development/Libraries
-Requires:       vitastor-client = %{version}-%{release}
-Requires:       fio = 3.27-7.el9
-
-
-%description -n vitastor-fio
-Vitastor fio drivers for benchmarking.
-
-
-%prep
-%setup -q
-
-
-%build
-%cmake
-%cmake_build
-
-
-%install
-rm -rf $RPM_BUILD_ROOT
-%cmake_install
-cd mon
-npm install
-cd ..
-mkdir -p %buildroot/usr/lib/vitastor
-cp -r mon %buildroot/usr/lib/vitastor
-mkdir -p %buildroot/lib/systemd/system
-cp mon/vitastor.target mon/vitastor-mon.service mon/vitastor-osd@.service %buildroot/lib/systemd/system
-mkdir -p %buildroot/lib/udev/rules.d
-cp mon/90-vitastor.rules %buildroot/lib/udev/rules.d
-
-
-%files
-%doc GPL-2.0.txt VNPL-1.1.txt README.md README-ru.md
-
-
-%files -n vitastor-osd
-%_bindir/vitastor-osd
-%_bindir/vitastor-disk
-%_bindir/vitastor-dump-journal
-/lib/systemd/system/vitastor-osd@.service
-/lib/systemd/system/vitastor.target
-/lib/udev/rules.d/90-vitastor.rules
-
-
-%pre -n vitastor-osd
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-install -o vitastor -g vitastor -d /var/log/vitastor
-mkdir -p /etc/vitastor
-
-
-%files -n vitastor-mon
-/usr/lib/vitastor/mon
-/lib/systemd/system/vitastor-mon.service
-
-
-%pre -n vitastor-mon
-groupadd -r -f vitastor 2>/dev/null ||:
-useradd -r -g vitastor -s /sbin/nologin -c "Vitastor daemons" -M -d /nonexistent vitastor 2>/dev/null ||:
-mkdir -p /etc/vitastor
-
-
-%files -n vitastor-client
-%_bindir/vitastor-nbd
-%_bindir/vitastor-nfs
-%_bindir/vitastor-cli
-%_bindir/vitastor-rm
-%_bindir/vita
-%_libdir/libvitastor_blk.so*
-%_libdir/libvitastor_client.so*
-
-
-%files -n vitastor-client-devel
-%_includedir/vitastor_c.h
-%_libdir/pkgconfig
-
-
-%files -n vitastor-fio
-%_libdir/libfio_vitastor.so
-%_libdir/libfio_vitastor_blk.so
-%_libdir/libfio_vitastor_sec.so
-
-
-%changelog

src/CMakeLists.txt

@@ -1,9 +1,8 @@
-cmake_minimum_required(VERSION 2.8.12)
+cmake_minimum_required(VERSION 2.8)
 
 project(vitastor)
 
 include(GNUInstallDirs)
-include(CTest)
 
 set(WITH_QEMU false CACHE BOOL "Build QEMU driver inside Vitastor source tree")
 set(WITH_FIO true CACHE BOOL "Build FIO driver")
@@ -16,7 +15,7 @@ if("${CMAKE_INSTALL_PREFIX}" MATCHES "^/usr/local/?$")
 	set(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_LIBDIR}")
 endif()
 
-add_definitions(-DVERSION="0.8.9")
+add_definitions(-DVERSION="0.8.3")
 add_definitions(-Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fdiagnostics-color=always -I ${CMAKE_SOURCE_DIR}/src)
 if (${WITH_ASAN})
 	add_definitions(-fsanitize=address -fno-omit-frame-pointer)
@@ -56,14 +55,6 @@ if (ISAL_LIBRARIES)
 	add_definitions(-DWITH_ISAL)
 endif (ISAL_LIBRARIES)
 
-add_custom_target(build_tests)
-add_custom_target(test
-	COMMAND
-	echo leak:tcmalloc > ${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt &&
-	env LSAN_OPTIONS=suppressions=${CMAKE_CURRENT_BINARY_DIR}/lsan-suppress.txt ${CMAKE_CTEST_COMMAND}
-)
-add_dependencies(test build_tests)
-
 include_directories(
 	../
 	/usr/include/jerasure
@@ -154,6 +145,7 @@ add_library(vitastor_client SHARED
 set_target_properties(vitastor_client PROPERTIES PUBLIC_HEADER "vitastor_c.h")
 target_link_libraries(vitastor_client
 	vitastor_common
+	tcmalloc_minimal
 	${LIBURING_LIBRARIES}
 	${IBVERBS_LIBRARIES}
 )
@@ -243,17 +235,14 @@ add_executable(osd_test osd_test.cpp rw_blocking.cpp addr_util.cpp)
 target_link_libraries(osd_test tcmalloc_minimal)
 
 # osd_rmw_test
-add_executable(osd_rmw_test EXCLUDE_FROM_ALL osd_rmw_test.cpp allocator.cpp)
+# FIXME: Move to tests
+add_executable(osd_rmw_test osd_rmw_test.cpp allocator.cpp)
 target_link_libraries(osd_rmw_test Jerasure ${ISAL_LIBRARIES} tcmalloc_minimal)
-add_dependencies(build_tests osd_rmw_test)
-add_test(NAME osd_rmw_test COMMAND osd_rmw_test)
 
 if (ISAL_LIBRARIES)
-	add_executable(osd_rmw_test_je EXCLUDE_FROM_ALL osd_rmw_test.cpp allocator.cpp)
+	add_executable(osd_rmw_test_je osd_rmw_test.cpp allocator.cpp)
 	target_compile_definitions(osd_rmw_test_je PUBLIC -DNO_ISAL)
 	target_link_libraries(osd_rmw_test_je Jerasure tcmalloc_minimal)
-	add_dependencies(build_tests osd_rmw_test_je)
-	add_test(NAME osd_rmw_test_jerasure COMMAND osd_rmw_test_je)
 endif (ISAL_LIBRARIES)
 
 # stub_uring_osd
@@ -268,15 +257,11 @@ target_link_libraries(stub_uring_osd
 )
 
 # osd_peering_pg_test
-add_executable(osd_peering_pg_test EXCLUDE_FROM_ALL osd_peering_pg_test.cpp osd_peering_pg.cpp)
+add_executable(osd_peering_pg_test osd_peering_pg_test.cpp osd_peering_pg.cpp)
 target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
-add_dependencies(build_tests osd_peering_pg_test)
-add_test(NAME osd_peering_pg_test COMMAND osd_peering_pg_test)
 
 # test_allocator
-add_executable(test_allocator EXCLUDE_FROM_ALL test_allocator.cpp allocator.cpp)
-add_dependencies(build_tests test_allocator)
-add_test(NAME test_allocator COMMAND test_allocator)
+add_executable(test_allocator test_allocator.cpp allocator.cpp)
 
 # test_cas
 add_executable(test_cas
@@ -296,15 +281,12 @@ target_link_libraries(test_crc32
 
 # test_cluster_client
 add_executable(test_cluster_client
-	EXCLUDE_FROM_ALL
 	test_cluster_client.cpp
 	pg_states.cpp osd_ops.cpp cluster_client.cpp cluster_client_list.cpp msgr_op.cpp mock/messenger.cpp msgr_stop.cpp
 	etcd_state_client.cpp timerfd_manager.cpp ../json11/json11.cpp
 )
 target_compile_definitions(test_cluster_client PUBLIC -D__MOCK__)
 target_include_directories(test_cluster_client PUBLIC ${CMAKE_SOURCE_DIR}/src/mock)
-add_dependencies(build_tests test_cluster_client)
-add_test(NAME test_cluster_client COMMAND test_cluster_client)
 
 ## test_blockstore, test_shit
 #add_executable(test_blockstore test_blockstore.cpp)

src/blockstore.cpp

@@ -13,11 +13,6 @@ blockstore_t::~blockstore_t()
     delete impl;
 }
 
-void blockstore_t::parse_config(blockstore_config_t & config)
-{
-    impl->parse_config(config, false);
-}
-
 void blockstore_t::loop()
 {
     impl->loop();

src/blockstore.h

@@ -107,7 +107,7 @@ Input:
 - buf = pre-allocated obj_ver_id array <len> units long
 
 Output:
-- retval = 0 or negative error number (-ENOENT if no such version for stabilize)
+- retval = 0 or negative error number (-EINVAL, -ENOENT if no such version or -EBUSY if not synced)
 
 ## BS_OP_SYNC_STAB_ALL
 
@@ -165,9 +165,6 @@ public:
     blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
     ~blockstore_t();
 
-    // Update configuration
-    void parse_config(blockstore_config_t & config);
-
     // Event loop
     void loop();
 

src/blockstore_flush.cpp

@@ -932,7 +932,7 @@ bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
     resume_1:
         if (!cur_sync->state)
         {
-            if (flusher->syncing_flushers >= flusher->active_flushers || !flusher->flush_queue.size())
+            if (flusher->syncing_flushers >= flusher->cur_flusher_count || !flusher->flush_queue.size())
             {
                 // Sync batch is ready. Do it.
                 await_sqe(0);

src/blockstore_impl.cpp

@@ -11,7 +11,7 @@ blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *
     ring_consumer.loop = [this]() { loop(); };
     ringloop->register_consumer(&ring_consumer);
     initialized = 0;
-    parse_config(config, true);
+    parse_config(config);
     zero_object = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, dsk.data_block_size);
     try
     {
@@ -171,7 +171,7 @@ void blockstore_impl_t::loop()
                     // Can't submit SYNC before previous writes
                     continue;
                 }
-                wr_st = continue_sync(op);
+                wr_st = continue_sync(op, false);
                 if (wr_st != 2)
                 {
                     has_writes = wr_st > 0 ? 1 : 2;
@@ -307,18 +307,6 @@ void blockstore_impl_t::check_wait(blockstore_op_t *op)
         }
         PRIV(op)->wait_for = 0;
     }
-    else if (PRIV(op)->wait_for == WAIT_FREE)
-    {
-        if (!data_alloc->get_free_count() && big_to_flush > 0)
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Still waiting for free space on the data device\n");
-#endif
-            return;
-        }
-        flusher->release_trim();
-        PRIV(op)->wait_for = 0;
-    }
     else
     {
         throw std::runtime_error("BUG: op->wait_for value is unexpected");
@@ -337,7 +325,7 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
     {
         // Basic verification not passed
         op->retval = -EINVAL;
-        ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });
+        std::function<void (blockstore_op_t*)>(op->callback)(op);
         return;
     }
     if (op->opcode == BS_OP_SYNC_STAB_ALL)
@@ -380,21 +368,16 @@ void blockstore_impl_t::enqueue_op(blockstore_op_t *op)
     }
     if ((op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE || op->opcode == BS_OP_DELETE) && !enqueue_write(op))
     {
-        ringloop->set_immediate([op]() { std::function<void (blockstore_op_t*)>(op->callback)(op); });
+        std::function<void (blockstore_op_t*)>(op->callback)(op);
         return;
     }
-    init_op(op);
-    submit_queue.push_back(op);
-    ringloop->wakeup();
-}
-
-void blockstore_impl_t::init_op(blockstore_op_t *op)
-{
     // Call constructor without allocating memory. We'll call destructor before returning op back
     new ((void*)op->private_data) blockstore_op_private_t;
     PRIV(op)->wait_for = 0;
     PRIV(op)->op_state = 0;
     PRIV(op)->pending_ops = 0;
+    submit_queue.push_back(op);
+    ringloop->wakeup();
 }
 
 static bool replace_stable(object_id oid, uint64_t version, int search_start, int search_end, obj_ver_id* list)

src/blockstore_impl.h

@@ -160,8 +160,6 @@ struct __attribute__((__packed__)) dirty_entry
 #define WAIT_JOURNAL 3
 // Suspend operation until the next journal sector buffer is free
 #define WAIT_JOURNAL_BUFFER 4
-// Suspend operation until there is some free space on the data device
-#define WAIT_FREE 5
 
 struct fulfill_read_t
 {
@@ -218,11 +216,6 @@ struct pool_shard_settings_t
     uint32_t pg_stripe_size;
 };
 
-#define STAB_SPLIT_DONE 1
-#define STAB_SPLIT_WAIT 2
-#define STAB_SPLIT_SYNC 3
-#define STAB_SPLIT_TODO 4
-
 class blockstore_impl_t
 {
     blockstore_disk_t dsk;
@@ -265,7 +258,6 @@ class blockstore_impl_t
 
     struct journal_t journal;
     journal_flusher_t *flusher;
-    int big_to_flush = 0;
     int write_iodepth = 0;
 
     bool live = false, queue_stall = false;
@@ -285,6 +277,7 @@ class blockstore_impl_t
     friend class journal_flusher_t;
     friend class journal_flusher_co;
 
+    void parse_config(blockstore_config_t & config);
     void calc_lengths();
     void open_data();
     void open_meta();
@@ -306,7 +299,6 @@ class blockstore_impl_t
     blockstore_init_journal* journal_init_reader;
 
     void check_wait(blockstore_op_t *op);
-    void init_op(blockstore_op_t *op);
 
     // Read
     int dequeue_read(blockstore_op_t *read_op);
@@ -326,7 +318,7 @@ class blockstore_impl_t
     void handle_write_event(ring_data_t *data, blockstore_op_t *op);
 
     // Sync
-    int continue_sync(blockstore_op_t *op);
+    int continue_sync(blockstore_op_t *op, bool queue_has_in_progress_sync);
     void ack_sync(blockstore_op_t *op);
 
     // Stabilize
@@ -334,8 +326,6 @@ class blockstore_impl_t
     int continue_stable(blockstore_op_t *op);
     void mark_stable(const obj_ver_id & ov, bool forget_dirty = false);
     void stabilize_object(object_id oid, uint64_t max_ver);
-    blockstore_op_t* selective_sync(blockstore_op_t *op);
-    int split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider);
 
     // Rollback
     int dequeue_rollback(blockstore_op_t *op);
@@ -351,8 +341,6 @@ public:
     blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop, timerfd_manager_t *tfd);
     ~blockstore_impl_t();
 
-    void parse_config(blockstore_config_t & config, bool init);
-
     // Event loop
     void loop();
 

src/blockstore_open.cpp

@@ -4,54 +4,8 @@
 #include <sys/file.h>
 #include "blockstore_impl.h"
 
-void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
+void blockstore_impl_t::parse_config(blockstore_config_t & config)
 {
-    // Online-configurable options:
-    max_flusher_count = strtoull(config["max_flusher_count"].c_str(), NULL, 10);
-    if (!max_flusher_count)
-    {
-        max_flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
-    }
-    min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10);
-    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
-    throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes";
-    throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10);
-    throttle_target_mbs = strtoull(config["throttle_target_mbs"].c_str(), NULL, 10);
-    throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
-    throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
-    if (!max_flusher_count)
-    {
-        max_flusher_count = 256;
-    }
-    if (!min_flusher_count || journal.flush_journal)
-    {
-        min_flusher_count = 1;
-    }
-    if (!max_write_iodepth)
-    {
-        max_write_iodepth = 128;
-    }
-    if (!throttle_target_iops)
-    {
-        throttle_target_iops = 100;
-    }
-    if (!throttle_target_mbs)
-    {
-        throttle_target_mbs = 100;
-    }
-    if (!throttle_target_parallelism)
-    {
-        throttle_target_parallelism = 1;
-    }
-    if (!throttle_threshold_us)
-    {
-        throttle_threshold_us = 50;
-    }
-    if (!init)
-    {
-        return;
-    }
-    // Offline-configurable options:
     // Common disk options
     dsk.parse_config(config);
     // Parse
@@ -90,7 +44,29 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     journal.no_same_sector_overwrites = config["journal_no_same_sector_overwrites"] == "true" ||
         config["journal_no_same_sector_overwrites"] == "1" || config["journal_no_same_sector_overwrites"] == "yes";
     journal.inmemory = config["inmemory_journal"] != "false";
+    max_flusher_count = strtoull(config["max_flusher_count"].c_str(), NULL, 10);
+    if (!max_flusher_count)
+        max_flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
+    min_flusher_count = strtoull(config["min_flusher_count"].c_str(), NULL, 10);
+    max_write_iodepth = strtoull(config["max_write_iodepth"].c_str(), NULL, 10);
+    throttle_small_writes = config["throttle_small_writes"] == "true" || config["throttle_small_writes"] == "1" || config["throttle_small_writes"] == "yes";
+    throttle_target_iops = strtoull(config["throttle_target_iops"].c_str(), NULL, 10);
+    throttle_target_mbs = strtoull(config["throttle_target_mbs"].c_str(), NULL, 10);
+    throttle_target_parallelism = strtoull(config["throttle_target_parallelism"].c_str(), NULL, 10);
+    throttle_threshold_us = strtoull(config["throttle_threshold_us"].c_str(), NULL, 10);
     // Validate
+    if (!max_flusher_count)
+    {
+        max_flusher_count = 256;
+    }
+    if (!min_flusher_count || journal.flush_journal)
+    {
+        min_flusher_count = 1;
+    }
+    if (!max_write_iodepth)
+    {
+        max_write_iodepth = 128;
+    }
     if (journal.sector_count < 2)
     {
         journal.sector_count = 32;
@@ -115,6 +91,22 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config, bool init)
     {
         throw std::runtime_error("immediate_commit=all requires disable_journal_fsync and disable_data_fsync");
     }
+    if (!throttle_target_iops)
+    {
+        throttle_target_iops = 100;
+    }
+    if (!throttle_target_mbs)
+    {
+        throttle_target_mbs = 100;
+    }
+    if (!throttle_target_parallelism)
+    {
+        throttle_target_parallelism = 1;
+    }
+    if (!throttle_threshold_us)
+    {
+        throttle_threshold_us = 50;
+    }
     // init some fields
     journal.block_size = dsk.journal_block_size;
     journal.next_free = dsk.journal_block_size;

src/blockstore_rollback.cpp

@@ -9,39 +9,48 @@ int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
     {
         return continue_rollback(op);
     }
-    int r = split_stab_op(op, [this](obj_ver_id ov)
+    obj_ver_id *v, *nv;
+    int i, todo = op->len;
+    for (i = 0, v = (obj_ver_id*)op->buf, nv = (obj_ver_id*)op->buf; i < op->len; i++, v++, nv++)
     {
+        if (nv != v)
+        {
+            *nv = *v;
+        }
         // Check that there are some versions greater than v->version (which may be zero),
         // check that they're unstable, synced, and not currently written to
         auto dirty_it = dirty_db.lower_bound((obj_ver_id){
-            .oid = ov.oid,
+            .oid = v->oid,
             .version = UINT64_MAX,
         });
         if (dirty_it == dirty_db.begin())
         {
+skip_ov:
             // Already rolled back, skip this object version
-            return STAB_SPLIT_DONE;
+            todo--;
+            nv--;
+            continue;
         }
         else
         {
             dirty_it--;
-            if (dirty_it->first.oid != ov.oid || dirty_it->first.version < ov.version)
+            if (dirty_it->first.oid != v->oid || dirty_it->first.version < v->version)
             {
-                // Already rolled back, skip this object version
-                return STAB_SPLIT_DONE;
+                goto skip_ov;
             }
-            while (dirty_it->first.oid == ov.oid && dirty_it->first.version > ov.version)
+            while (dirty_it->first.oid == v->oid && dirty_it->first.version > v->version)
             {
                 if (IS_IN_FLIGHT(dirty_it->second.state))
                 {
                     // Object write is still in progress. Wait until the write request completes
-                    return STAB_SPLIT_WAIT;
+                    return 0;
                 }
                 else if (!IS_SYNCED(dirty_it->second.state) ||
                     IS_STABLE(dirty_it->second.state))
                 {
-                    // Sync the object
-                    return STAB_SPLIT_SYNC;
+                    op->retval = -EBUSY;
+                    FINISH_OP(op);
+                    return 2;
                 }
                 if (dirty_it == dirty_db.begin())
                 {
@@ -49,16 +58,19 @@ int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
                 }
                 dirty_it--;
             }
-            return STAB_SPLIT_TODO;
         }
-    });
-    if (r != 1)
+    }
+    op->len = todo;
+    if (!todo)
     {
-        return r;
+        // Already rolled back
+        op->retval = 0;
+        FINISH_OP(op);
+        return 2;
     }
     // Check journal space
     blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, op->len, sizeof(journal_entry_rollback), 0))
+    if (!space_check.check_available(op, todo, sizeof(journal_entry_rollback), 0))
     {
         return 0;
     }
@@ -66,8 +78,7 @@ int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
     BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
     // Prepare and submit journal entries
     int s = 0;
-    auto v = (obj_ver_id*)op->buf;
-    for (int i = 0; i < op->len; i++, v++)
+    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
     {
         if (!journal.entry_fits(sizeof(journal_entry_rollback)) &&
             journal.sector_info[journal.cur_sector].dirty)
@@ -201,11 +212,6 @@ void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start,
     }
     while (1)
     {
-        if ((IS_BIG_WRITE(dirty_it->second.state) || IS_DELETE(dirty_it->second.state)) &&
-            IS_STABLE(dirty_it->second.state))
-        {
-            big_to_flush--;
-        }
         if (IS_BIG_WRITE(dirty_it->second.state) && dirty_it->second.location != clean_loc &&
             dirty_it->second.location != UINT64_MAX)
         {

src/blockstore_stable.cpp

@@ -41,309 +41,60 @@
 // 4) after a while it takes his synced object list and sends stabilize requests
 //    to peers and to its own blockstore, thus freeing the old version
 
-struct ver_vector_t
-{
-    obj_ver_id *items = NULL;
-    uint64_t alloc = 0, size = 0;
-};
-
-static void init_versions(ver_vector_t & vec, obj_ver_id *start, obj_ver_id *end, uint64_t len)
-{
-    if (!vec.items)
-    {
-        vec.alloc = len;
-        vec.items = (obj_ver_id*)malloc_or_die(sizeof(obj_ver_id) * vec.alloc);
-        for (auto sv = start; sv < end; sv++)
-        {
-            vec.items[vec.size++] = *sv;
-        }
-    }
-}
-
-static void append_version(ver_vector_t & vec, obj_ver_id ov)
-{
-    if (vec.size >= vec.alloc)
-    {
-        vec.alloc = !vec.alloc ? 4 : vec.alloc*2;
-        vec.items = (obj_ver_id*)realloc_or_die(vec.items, sizeof(obj_ver_id) * vec.alloc);
-    }
-    vec.items[vec.size++] = ov;
-}
-
-static bool check_unsynced(std::vector<obj_ver_id> & check, obj_ver_id ov, std::vector<obj_ver_id> & to, int *count)
-{
-    bool found = false;
-    int j = 0, k = 0;
-    while (j < check.size())
-    {
-        if (check[j] == ov)
-            found = true;
-        if (check[j].oid == ov.oid && check[j].version <= ov.version)
-        {
-            to.push_back(check[j++]);
-            if (count)
-                (*count)--;
-        }
-        else
-            check[k++] = check[j++];
-    }
-    check.resize(k);
-    return found;
-}
-
-blockstore_op_t* blockstore_impl_t::selective_sync(blockstore_op_t *op)
-{
-    unsynced_big_write_count -= unsynced_big_writes.size();
-    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
-    unsynced_big_write_count += unsynced_big_writes.size();
-    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
-    // Create a sync operation, insert into the end of the queue
-    // And move ourselves into the end too!
-    // Rather hacky but that's what we need...
-    blockstore_op_t *sync_op = new blockstore_op_t;
-    sync_op->opcode = BS_OP_SYNC;
-    sync_op->buf = NULL;
-    sync_op->callback = [this](blockstore_op_t *sync_op)
-    {
-        delete sync_op;
-    };
-    init_op(sync_op);
-    int sync_res = continue_sync(sync_op);
-    if (sync_res != 2)
-    {
-        // Put SYNC into the queue if it's not finished yet
-        submit_queue.push_back(sync_op);
-    }
-    // Restore unsynced_writes
-    unsynced_small_writes.swap(PRIV(op)->sync_small_writes);
-    unsynced_big_write_count -= unsynced_big_writes.size();
-    unsynced_big_writes.swap(PRIV(op)->sync_big_writes);
-    unsynced_big_write_count += unsynced_big_writes.size();
-    if (sync_res == 2)
-    {
-        // Sync is immediately completed
-        return NULL;
-    }
-    return sync_op;
-}
-
-// Returns: 2 = stop processing and dequeue, 0 = stop processing and do not dequeue, 1 = proceed with op itself
-int blockstore_impl_t::split_stab_op(blockstore_op_t *op, std::function<int(obj_ver_id v)> decider)
-{
-    bool add_sync = false;
-    ver_vector_t good_vers, bad_vers;
-    obj_ver_id* v;
-    int i, todo = 0;
-    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-    {
-        int action = decider(*v);
-        if (action < 0)
-        {
-            // Rollback changes
-            for (auto & ov: PRIV(op)->sync_big_writes)
-            {
-                unsynced_big_writes.push_back(ov);
-                unsynced_big_write_count++;
-            }
-            for (auto & ov: PRIV(op)->sync_small_writes)
-            {
-                unsynced_small_writes.push_back(ov);
-            }
-            free(good_vers.items);
-            good_vers.items = NULL;
-            free(bad_vers.items);
-            bad_vers.items = NULL;
-            // Error
-            op->retval = action;
-            FINISH_OP(op);
-            return 2;
-        }
-        else if (action == STAB_SPLIT_DONE)
-        {
-            // Already done
-            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
-        }
-        else if (action == STAB_SPLIT_WAIT)
-        {
-            // Already in progress, we just have to wait until it finishes
-            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
-            append_version(bad_vers, *v);
-        }
-        else if (action == STAB_SPLIT_SYNC)
-        {
-            // Needs a SYNC, we have to send a SYNC if not already in progress
-            //
-            // If the object is not present in unsynced_(big|small)_writes then
-            // it's currently being synced. If it's present then we can initiate
-            // its sync ourselves.
-            init_versions(good_vers, (obj_ver_id*)op->buf, v, op->len);
-            append_version(bad_vers, *v);
-            if (!add_sync)
-            {
-                PRIV(op)->sync_big_writes.clear();
-                PRIV(op)->sync_small_writes.clear();
-                add_sync = true;
-            }
-            check_unsynced(unsynced_small_writes, *v, PRIV(op)->sync_small_writes, NULL);
-            check_unsynced(unsynced_big_writes, *v, PRIV(op)->sync_big_writes, &unsynced_big_write_count);
-        }
-        else /* if (action == STAB_SPLIT_TODO) */
-        {
-            if (good_vers.items)
-            {
-                // If we're selecting versions then append it
-                // Main idea is that 99% of the time all versions passed to BS_OP_STABLE are synced
-                // And we don't want to select/allocate anything in that optimistic case
-                append_version(good_vers, *v);
-            }
-            todo++;
-        }
-    }
-    // In a pessimistic scenario, an operation may be split into 3:
-    // - Stabilize synced entries
-    // - Sync unsynced entries
-    // - Continue for unsynced entries after sync
-    add_sync = add_sync && (PRIV(op)->sync_big_writes.size() || PRIV(op)->sync_small_writes.size());
-    if (!todo && !bad_vers.size)
-    {
-        // Already stable
-        op->retval = 0;
-        FINISH_OP(op);
-        return 2;
-    }
-    op->retval = 0;
-    if (!todo && !add_sync)
-    {
-        // Only wait for inflight writes or current in-progress syncs
-        return 0;
-    }
-    blockstore_op_t *sync_op = NULL, *split_stab_op = NULL;
-    if (add_sync)
-    {
-        // Initiate a selective sync for PRIV(op)->sync_(big|small)_writes
-        sync_op = selective_sync(op);
-    }
-    if (bad_vers.size)
-    {
-        // Split part of the request into a separate operation
-        split_stab_op = new blockstore_op_t;
-        split_stab_op->opcode = op->opcode;
-        split_stab_op->buf = bad_vers.items;
-        split_stab_op->len = bad_vers.size;
-        init_op(split_stab_op);
-        submit_queue.push_back(split_stab_op);
-    }
-    if (sync_op || split_stab_op || good_vers.items)
-    {
-        void *orig_buf = op->buf;
-        if (good_vers.items)
-        {
-            op->buf = good_vers.items;
-            op->len = good_vers.size;
-        }
-        // Make a wrapped callback
-        int *split_op_counter = (int*)malloc_or_die(sizeof(int));
-        *split_op_counter = (sync_op ? 1 : 0) + (split_stab_op ? 1 : 0) + (todo ? 1 : 0);
-        auto cb = [this, op, good_items = good_vers.items,
-            bad_items = bad_vers.items, split_op_counter,
-            orig_buf, real_cb = op->callback](blockstore_op_t *split_op)
-        {
-            if (split_op->retval != 0)
-                op->retval = split_op->retval;
-            (*split_op_counter)--;
-            assert((*split_op_counter) >= 0);
-            if (op != split_op)
-                delete split_op;
-            if (!*split_op_counter)
-            {
-                free(good_items);
-                free(bad_items);
-                free(split_op_counter);
-                op->buf = orig_buf;
-                real_cb(op);
-            }
-        };
-        if (sync_op)
-        {
-            sync_op->callback = cb;
-        }
-        if (split_stab_op)
-        {
-            split_stab_op->callback = cb;
-        }
-        op->callback = cb;
-    }
-    if (!todo)
-    {
-        // All work is postponed
-        op->callback = NULL;
-        return 2;
-    }
-    return 1;
-}
-
 int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
 {
     if (PRIV(op)->op_state)
     {
         return continue_stable(op);
     }
-    int r = split_stab_op(op, [this](obj_ver_id ov)
+    obj_ver_id* v;
+    int i, todo = 0;
+    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
     {
-        auto dirty_it = dirty_db.find(ov);
+        auto dirty_it = dirty_db.find(*v);
         if (dirty_it == dirty_db.end())
         {
-            auto & clean_db = clean_db_shard(ov.oid);
-            auto clean_it = clean_db.find(ov.oid);
-            if (clean_it == clean_db.end() || clean_it->second.version < ov.version)
+            auto & clean_db = clean_db_shard(v->oid);
+            auto clean_it = clean_db.find(v->oid);
+            if (clean_it == clean_db.end() || clean_it->second.version < v->version)
             {
                 // No such object version
-                printf("Error: %lx:%lx v%lu not found while stabilizing\n", ov.oid.inode, ov.oid.stripe, ov.version);
-                return -ENOENT;
+                op->retval = -ENOENT;
+                FINISH_OP(op);
+                return 2;
             }
             else
             {
                 // Already stable
-                return STAB_SPLIT_DONE;
             }
         }
         else if (IS_IN_FLIGHT(dirty_it->second.state))
         {
             // Object write is still in progress. Wait until the write request completes
-            return STAB_SPLIT_WAIT;
+            return 0;
         }
         else if (!IS_SYNCED(dirty_it->second.state))
         {
-            // Object not synced yet - sync it
-            // In previous versions we returned EBUSY here and required
-            // the caller (OSD) to issue a global sync first. But a global sync
-            // waits for all writes in the queue including inflight writes. And
-            // inflight writes may themselves be blocked by unstable writes being
-            // still present in the journal and not flushed away from it.
-            // So we must sync specific objects here.
-            //
-            // Even more, we have to process "stabilize" request in parts. That is,
-            // we must stabilize all objects which are already synced. Otherwise
-            // they may block objects which are NOT synced yet.
-            return STAB_SPLIT_SYNC;
+            // Object not synced yet. Caller must sync it first
+            op->retval = -EBUSY;
+            FINISH_OP(op);
+            return 2;
         }
-        else if (IS_STABLE(dirty_it->second.state))
+        else if (!IS_STABLE(dirty_it->second.state))
         {
-            // Already stable
-            return STAB_SPLIT_DONE;
+            todo++;
         }
-        else
-        {
-            return STAB_SPLIT_TODO;
-        }
-    });
-    if (r != 1)
+    }
+    if (!todo)
     {
-        return r;
+        // Already stable
+        op->retval = 0;
+        FINISH_OP(op);
+        return 2;
     }
     // Check journal space
     blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, op->len, sizeof(journal_entry_stable), 0))
+    if (!space_check.check_available(op, todo, sizeof(journal_entry_stable), 0))
     {
         return 0;
     }
@@ -351,9 +102,9 @@ int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
     BS_SUBMIT_CHECK_SQES(space_check.sectors_to_write);
     // Prepare and submit journal entries
     int s = 0;
-    auto v = (obj_ver_id*)op->buf;
-    for (int i = 0; i < op->len; i++, v++)
+    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
     {
+        // FIXME: Only stabilize versions that aren't stable yet
         if (!journal.entry_fits(sizeof(journal_entry_stable)) &&
             journal.sector_info[journal.cur_sector].dirty)
         {
@@ -446,7 +197,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
                     {
                         inode_space_stats[dirty_it->first.oid.inode] += dsk.data_block_size;
                     }
-                    big_to_flush++;
                 }
                 else if (IS_DELETE(dirty_it->second.state))
                 {
@@ -455,7 +205,6 @@ void blockstore_impl_t::mark_stable(const obj_ver_id & v, bool forget_dirty)
                         sp -= dsk.data_block_size;
                     else
                         inode_space_stats.erase(dirty_it->first.oid.inode);
-                    big_to_flush++;
                 }
             }
             if (forget_dirty && (IS_BIG_WRITE(dirty_it->second.state) ||

src/blockstore_sync.cpp

@@ -12,7 +12,7 @@
 #define SYNC_JOURNAL_SYNC_SENT 7
 #define SYNC_DONE 8
 
-int blockstore_impl_t::continue_sync(blockstore_op_t *op)
+int blockstore_impl_t::continue_sync(blockstore_op_t *op, bool queue_has_in_progress_sync)
 {
     if (immediate_commit == IMMEDIATE_ALL)
     {
@@ -145,7 +145,7 @@ int blockstore_impl_t::continue_sync(blockstore_op_t *op)
             PRIV(op)->op_state = SYNC_DONE;
         }
     }
-    if (PRIV(op)->op_state == SYNC_DONE)
+    if (PRIV(op)->op_state == SYNC_DONE && !queue_has_in_progress_sync)
     {
         ack_sync(op);
         return 2;

src/blockstore_write.cpp

@@ -271,13 +271,6 @@ int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
         if (loc == UINT64_MAX)
         {
             // no space
-            if (big_to_flush > 0)
-            {
-                // hope that some space will be available after flush
-                flusher->request_trim();
-                PRIV(op)->wait_for = WAIT_FREE;
-                return 0;
-            }
             cancel_all_writes(op, dirty_it, -ENOSPC);
             return 2;
         }

src/cli_df.cpp

@@ -121,7 +121,8 @@ resume_1:
             }
             if (pool_cfg.scheme != POOL_SCHEME_REPLICATED)
             {
-                pool_avail *= (pool_cfg.pg_size - pool_cfg.parity_chunks);
+                uint64_t pg_real_size = pool_stats[pool_cfg.id]["pg_real_size"].uint64_value();
+                pool_avail = pg_real_size > 0 ? pool_avail * (pool_cfg.pg_size - pool_cfg.parity_chunks) / pg_real_size : 0;
             }
             pool_stats[pool_cfg.id] = json11::Json::object {
                 { "name", pool_cfg.name },

src/cli_merge.cpp

@@ -403,7 +403,7 @@ struct snap_merger_t
         op->opcode = OSD_OP_READ_BITMAP;
         op->inode = target;
         op->offset = offset;
-        op->len = target_block_size;
+        op->len = 0;
         op->callback = [this](cluster_op_t *op)
         {
             if (op->retval < 0)

src/cli_rm_data.cpp

@@ -92,7 +92,6 @@ struct rm_inode_t
 
     void send_ops(rm_pg_t *cur_list)
     {
-        parent->cli->init_msgr();
         if (parent->cli->msgr.osd_peer_fds.find(cur_list->rm_osd_num) ==
             parent->cli->msgr.osd_peer_fds.end())
         {

src/cli_rm_osd.cpp

@@ -278,7 +278,7 @@ struct rm_osd_t
             if (rsp["response_delete_range"]["deleted"].uint64_value() > 0)
             {
                 // Wait for mon_change_timeout before updating PG history, or the monitor's change will likely interfere with ours
-                retry_wait = parent->cli->config["mon_change_timeout"].uint64_value();
+                retry_wait = parent->cli->merged_config["mon_change_timeout"].uint64_value();
                 if (!retry_wait)
                     retry_wait = 1000;
                 retry_wait += etcd_tx_retry_ms;

src/cli_status.cpp

@@ -198,9 +198,9 @@ resume_2:
             }
             pgs_by_state_str += std::to_string(kv.second)+" "+kv.first;
         }
-        bool readonly = json_is_true(parent->cli->config["readonly"]);
-        bool no_recovery = json_is_true(parent->cli->config["no_recovery"]);
-        bool no_rebalance = json_is_true(parent->cli->config["no_rebalance"]);
+        bool readonly = json_is_true(parent->cli->merged_config["readonly"]);
+        bool no_recovery = json_is_true(parent->cli->merged_config["no_recovery"]);
+        bool no_rebalance = json_is_true(parent->cli->merged_config["no_rebalance"]);
         if (parent->json_output)
         {
             // JSON output

src/cluster_client.cpp

@@ -18,12 +18,11 @@
 
 cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config)
 {
-    cli_config = config.object_items();
-    file_config = osd_messenger_t::read_config(config);
-    config = osd_messenger_t::merge_configs(cli_config, file_config, etcd_global_config, {});
+    config = osd_messenger_t::read_config(config);
 
     this->ringloop = ringloop;
     this->tfd = tfd;
+    this->config = config;
 
     msgr.osd_num = 0;
     msgr.tfd = tfd;
@@ -59,7 +58,8 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
         msgr.stop_client(op->peer_fd);
         delete op;
     };
-    msgr.parse_config(config);
+    msgr.parse_config(this->config);
+    msgr.init();
 
     st_cli.tfd = tfd;
     st_cli.on_load_config_hook = [this](json11::Json::object & cfg) { on_load_config_hook(cfg); };
@@ -73,6 +73,17 @@ cluster_client_t::cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd
 
     scrap_buffer_size = SCRAP_BUFFER_SIZE;
     scrap_buffer = malloc_or_die(scrap_buffer_size);
+
+    if (ringloop)
+    {
+        consumer.loop = [this]()
+        {
+            msgr.read_requests();
+            msgr.send_replies();
+            this->ringloop->submit();
+        };
+        ringloop->register_consumer(&consumer);
+    }
 }
 
 cluster_client_t::~cluster_client_t()
@@ -104,24 +115,6 @@ cluster_op_t::~cluster_op_t()
     }
 }
 
-void cluster_client_t::init_msgr()
-{
-    if (msgr_initialized)
-        return;
-    msgr.init();
-    msgr_initialized = true;
-    if (ringloop)
-    {
-        consumer.loop = [this]()
-        {
-            msgr.read_requests();
-            msgr.send_replies();
-            this->ringloop->submit();
-        };
-        ringloop->register_consumer(&consumer);
-    }
-}
-
 void cluster_client_t::calc_wait(cluster_op_t *op)
 {
     op->prev_wait = 0;
@@ -150,7 +143,7 @@ void cluster_client_t::calc_wait(cluster_op_t *op)
         if (!op->prev_wait)
             continue_sync(op);
     }
-    else /* if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) */
+    else /* if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP) */
     {
         for (auto prev = op_queue_head; prev && prev != op; prev = prev->next)
         {
@@ -158,8 +151,7 @@ void cluster_client_t::calc_wait(cluster_op_t *op)
             {
                 op->prev_wait++;
             }
-            else if (prev->opcode == OSD_OP_WRITE || prev->opcode == OSD_OP_READ ||
-                prev->opcode == OSD_OP_READ_BITMAP || prev->opcode == OSD_OP_READ_CHAIN_BITMAP)
+            else if (prev->opcode == OSD_OP_WRITE || prev->opcode == OSD_OP_READ || prev->opcode == OSD_OP_READ_BITMAP)
             {
                 // Flushes are always in the beginning (we're scanning from the beginning of the queue)
                 break;
@@ -179,8 +171,7 @@ void cluster_client_t::inc_wait(uint64_t opcode, uint64_t flags, cluster_op_t *n
             auto n2 = next->next;
             if (next->opcode == OSD_OP_SYNC && !(flags & OP_IMMEDIATE_COMMIT) ||
                 next->opcode == OSD_OP_WRITE && (flags & OP_FLUSH_BUFFER) && !(next->flags & OP_FLUSH_BUFFER) ||
-                (next->opcode == OSD_OP_READ || next->opcode == OSD_OP_READ_BITMAP ||
-                    next->opcode == OSD_OP_READ_CHAIN_BITMAP) && (flags & OP_FLUSH_BUFFER))
+                (next->opcode == OSD_OP_READ || next->opcode == OSD_OP_READ_BITMAP) && (flags & OP_FLUSH_BUFFER))
             {
                 next->prev_wait += inc;
                 assert(next->prev_wait >= 0);
@@ -230,14 +221,11 @@ void cluster_client_t::erase_op(cluster_op_t *op)
     if (op_queue_tail == op)
         op_queue_tail = op->prev;
     op->next = op->prev = NULL;
-    if (flags & OP_FLUSH_BUFFER)
-        std::function<void(cluster_op_t*)>(op->callback)(op);
     if (!(flags & OP_IMMEDIATE_COMMIT))
         inc_wait(opcode, flags, next, -1);
     // Call callback at the end to avoid inconsistencies in prev_wait
     // if the callback adds more operations itself
-    if (!(flags & OP_FLUSH_BUFFER))
-        std::function<void(cluster_op_t*)>(op->callback)(op);
+    std::function<void(cluster_op_t*)>(op->callback)(op);
 }
 
 void cluster_client_t::continue_ops(bool up_retry)
@@ -277,10 +265,13 @@ restart:
     continuing_ops = 0;
 }
 
-void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_config)
+void cluster_client_t::on_load_config_hook(json11::Json::object & config)
 {
-    this->etcd_global_config = etcd_global_config;
-    config = osd_messenger_t::merge_configs(cli_config, file_config, etcd_global_config, {});
+    this->merged_config = config;
+    for (auto & kv: this->config.object_items())
+    {
+        this->merged_config[kv.first] = kv.second;
+    }
     if (config.find("client_max_dirty_bytes") != config.end())
     {
         client_max_dirty_bytes = config["client_max_dirty_bytes"].uint64_value();
@@ -290,13 +281,14 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
         // Old name
         client_max_dirty_bytes = config["client_dirty_limit"].uint64_value();
     }
-    else
-        client_max_dirty_bytes = 0;
+    if (config.find("client_max_dirty_ops") != config.end())
+    {
+        client_max_dirty_ops = config["client_max_dirty_ops"].uint64_value();
+    }
     if (!client_max_dirty_bytes)
     {
         client_max_dirty_bytes = DEFAULT_CLIENT_MAX_DIRTY_BYTES;
     }
-    client_max_dirty_ops = config["client_max_dirty_ops"].uint64_value();
     if (!client_max_dirty_ops)
     {
         client_max_dirty_ops = DEFAULT_CLIENT_MAX_DIRTY_OPS;
@@ -311,7 +303,7 @@ void cluster_client_t::on_load_config_hook(json11::Json::object & etcd_global_co
         up_wait_retry_interval = 50;
     }
     msgr.parse_config(config);
-    st_cli.parse_config(config);
+    msgr.parse_config(this->config);
     st_cli.load_pgs();
 }
 
@@ -345,8 +337,7 @@ void cluster_client_t::on_change_hook(std::map<std::string, etcd_kv_t> & changes
             // And now they have to be resliced!
             for (auto op = op_queue_head; op; op = op->next)
             {
-                if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_READ ||
-                    op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP) &&
+                if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP) &&
                     INODE_POOL(op->cur_inode) == pool_item.first)
                 {
                     op->needs_reslice = true;
@@ -418,7 +409,7 @@ void cluster_client_t::on_ready(std::function<void(void)> fn)
 void cluster_client_t::execute(cluster_op_t *op)
 {
     if (op->opcode != OSD_OP_SYNC && op->opcode != OSD_OP_READ &&
-        op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_WRITE)
+        op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_WRITE)
     {
         op->retval = -EINVAL;
         std::function<void(cluster_op_t*)>(op->callback)(op);
@@ -450,7 +441,7 @@ void cluster_client_t::execute(cluster_op_t *op)
             return;
         }
         // Check alignment
-        if (!op->len && (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP || op->opcode == OSD_OP_WRITE) ||
+        if ((op->opcode == OSD_OP_READ || op->opcode == OSD_OP_WRITE) && !op->len ||
             op->offset % pool_it->second.bitmap_granularity || op->len % pool_it->second.bitmap_granularity)
         {
             op->retval = -EINVAL;
@@ -711,7 +702,8 @@ resume_3:
         // Finished successfully
         // Even if the PG count has changed in meanwhile we treat it as success
         // because if some operations were invalid for the new PG count we'd get errors
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        bool is_read = op->opcode == OSD_OP_READ;
+        if (is_read)
         {
             // Check parent inode
             auto ino_it = st_cli.inode_config.find(op->cur_inode);
@@ -735,24 +727,18 @@ resume_3:
             }
         }
         op->retval = op->len;
-        if (op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
-        {
-            auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->inode));
-            op->retval = op->len / pool_cfg.bitmap_granularity;
-        }
         erase_op(op);
         return 1;
     }
-    else if (op->retval != 0 && op->retval != -EPIPE && op->retval != -EIO && op->retval != -ENOSPC)
+    else if (op->retval != 0 && op->retval != -EPIPE)
     {
-        // Fatal error (neither -EPIPE, -EIO nor -ENOSPC)
-        // FIXME: Add a parameter to allow to not wait for EIOs (incomplete or corrupted objects) to heal
+        // Fatal error (not -EPIPE)
         erase_op(op);
         return 1;
     }
     else
     {
-        // Non-fatal error - clear the error and retry
+        // -EPIPE - clear the error and retry
         op->retval = 0;
         if (op->needs_reslice)
         {
@@ -764,10 +750,7 @@ resume_3:
         {
             for (int i = 0; i < op->parts.size(); i++)
             {
-                if (!(op->parts[i].flags & PART_DONE))
-                {
-                    op->parts[i].flags = PART_RETRY;
-                }
+                op->parts[i].flags = PART_RETRY;
             }
             goto resume_2;
         }
@@ -826,19 +809,23 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
     uint64_t last_stripe = op->len > 0 ? ((op->offset + op->len - 1) / pg_block_size) * pg_block_size : first_stripe;
     op->retval = 0;
     op->parts.resize((last_stripe - first_stripe) / pg_block_size + 1);
-    if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+    if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP)
     {
         // Allocate memory for the bitmap
-        unsigned object_bitmap_size = ((op->len / pool_cfg.bitmap_granularity + 7) / 8);
+        unsigned object_bitmap_size = (((op->opcode == OSD_OP_READ_BITMAP ? pg_block_size : op->len) / pool_cfg.bitmap_granularity + 7) / 8);
         object_bitmap_size = (object_bitmap_size < 8 ? 8 : object_bitmap_size);
         unsigned bitmap_mem = object_bitmap_size + (pool_cfg.data_block_size / pool_cfg.bitmap_granularity / 8 * pg_data_size) * op->parts.size();
-        if (!op->bitmap_buf || op->bitmap_buf_size < bitmap_mem)
+        if (op->bitmap_buf_size < bitmap_mem)
         {
             op->bitmap_buf = realloc_or_die(op->bitmap_buf, bitmap_mem);
+            if (!op->bitmap_buf_size)
+            {
+                // First allocation
+                memset(op->bitmap_buf, 0, object_bitmap_size);
+            }
             op->part_bitmaps = (uint8_t*)op->bitmap_buf + object_bitmap_size;
             op->bitmap_buf_size = bitmap_mem;
         }
-        memset(op->bitmap_buf, 0, bitmap_mem);
     }
     int iov_idx = 0;
     size_t iov_pos = 0;
@@ -889,14 +876,13 @@ void cluster_client_t::slice_rw(cluster_op_t *op)
             if (end == begin)
                 op->done_count++;
         }
-        else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
+        else if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
         {
             add_iov(end-begin, false, op, iov_idx, iov_pos, op->parts[i].iov, NULL, 0);
         }
         op->parts[i].parent = op;
         op->parts[i].offset = begin;
-        op->parts[i].len = op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP ||
-            op->opcode == OSD_OP_DELETE ? 0 : (uint32_t)(end - begin);
+        op->parts[i].len = op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_DELETE ? 0 : (uint32_t)(end - begin);
         op->parts[i].pg_num = pg_num;
         op->parts[i].osd_num = 0;
         op->parts[i].flags = 0;
@@ -925,10 +911,6 @@ bool cluster_client_t::affects_osd(uint64_t inode, uint64_t offset, uint64_t len
 
 bool cluster_client_t::try_send(cluster_op_t *op, int i)
 {
-    if (!msgr_initialized)
-    {
-        init_msgr();
-    }
     auto part = &op->parts[i];
     auto & pool_cfg = st_cli.pool_config.at(INODE_POOL(op->cur_inode));
     auto pg_it = pool_cfg.pg_config.find(part->pg_num);
@@ -947,7 +929,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
                 pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks
             );
             uint64_t meta_rev = 0;
-            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_READ_CHAIN_BITMAP && op->opcode != OSD_OP_DELETE)
+            if (op->opcode != OSD_OP_READ_BITMAP && op->opcode != OSD_OP_DELETE)
             {
                 auto ino_it = st_cli.inode_config.find(op->inode);
                 if (ino_it != st_cli.inode_config.end())
@@ -960,7 +942,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
                     .header = {
                         .magic = SECONDARY_OSD_OP_MAGIC,
                         .id = next_op_id(),
-                        .opcode = op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP ? OSD_OP_READ : op->opcode,
+                        .opcode = op->opcode == OSD_OP_READ_BITMAP ? OSD_OP_READ : op->opcode,
                     },
                     .inode = op->cur_inode,
                     .offset = part->offset,
@@ -968,10 +950,8 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
                     .meta_revision = meta_rev,
                     .version = op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE ? op->version : 0,
                 } },
-                .bitmap = (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
-                    ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
-                .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP
-                    ? pg_bitmap_size : 0),
+                .bitmap = (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP ? (uint8_t*)op->part_bitmaps + pg_bitmap_size*i : NULL),
+                .bitmap_len = (unsigned)(op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP ? pg_bitmap_size : 0),
                 .callback = [this, part](osd_op_t *op_part)
                 {
                     handle_op_part(part);
@@ -1049,7 +1029,7 @@ resume_1:
                 uw_it->second.state = CACHE_DIRTY;
             }
         }
-        if (op->retval == -EPIPE || op->retval == -EIO || op->retval == -ENOSPC)
+        if (op->retval == -EPIPE)
         {
             // Retry later
             op->parts.clear();
@@ -1119,72 +1099,53 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
     if (part->op.reply.hdr.retval != expected)
     {
         // Operation failed, retry
-        part->flags |= PART_ERROR;
-        if (!op->retval || op->retval == -EPIPE || part->op.reply.hdr.retval == -EIO)
+        if (part->op.reply.hdr.retval == -EPIPE)
         {
-            // Error priority: EIO > ENOSPC > EPIPE
+            // Mark op->up_wait = true before stopping the client
+            op->up_wait = true;
+            if (!retry_timeout_id)
+            {
+                retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
+                {
+                    retry_timeout_id = 0;
+                    continue_ops(true);
+                });
+            }
+        }
+        if (!op->retval || op->retval == -EPIPE)
+        {
+            // Don't overwrite other errors with -EPIPE
             op->retval = part->op.reply.hdr.retval;
         }
-        int stop_fd = -1;
-        if (op->retval != -EINTR && op->retval != -EIO && op->retval != -ENOSPC)
+        if (op->retval != -EINTR && op->retval != -EIO)
         {
-            stop_fd = part->op.peer_fd;
             fprintf(
                 stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d), dropping connection\n",
                 osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected
             );
+            msgr.stop_client(part->op.peer_fd);
         }
-        else
-        {
-            fprintf(
-                stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d)\n",
-                osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected
-            );
-        }
-        // All next things like timer, continue_sync/rw and stop_client may affect the operation again
-        // So do all these things after modifying operation state, otherwise we may hit reenterability bugs
-        // FIXME postpone such things to set_immediate here to avoid bugs
-        // Mark op->up_wait = true to retry operation after a short pause (not immediately)
-        op->up_wait = true;
-        if (!retry_timeout_id)
-        {
-            retry_timeout_id = tfd->set_timer(up_wait_retry_interval, false, [this](int)
-            {
-                retry_timeout_id = 0;
-                continue_ops(true);
-            });
-        }
-        if (op->inflight_count == 0)
-        {
-            if (op->opcode == OSD_OP_SYNC)
-                continue_sync(op);
-            else
-                continue_rw(op);
-        }
-        if (stop_fd >= 0)
-        {
-            msgr.stop_client(stop_fd);
-        }
+        part->flags |= PART_ERROR;
     }
     else
     {
         // OK
-        if ((op->opcode == OSD_OP_WRITE || op->opcode == OSD_OP_DELETE) && !(op->flags & OP_IMMEDIATE_COMMIT))
+        if (!(op->flags & OP_IMMEDIATE_COMMIT))
             dirty_osds.insert(part->osd_num);
         part->flags |= PART_DONE;
         op->done_count++;
-        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP || op->opcode == OSD_OP_READ_CHAIN_BITMAP)
+        if (op->opcode == OSD_OP_READ || op->opcode == OSD_OP_READ_BITMAP)
         {
             copy_part_bitmap(op, part);
             op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
         }
-        if (op->inflight_count == 0)
-        {
-            if (op->opcode == OSD_OP_SYNC)
-                continue_sync(op);
-            else
-                continue_rw(op);
-        }
+    }
+    if (op->inflight_count == 0)
+    {
+        if (op->opcode == OSD_OP_SYNC)
+            continue_sync(op);
+        else
+            continue_rw(op);
     }
 }
 
@@ -1197,12 +1158,7 @@ void cluster_client_t::copy_part_bitmap(cluster_op_t *op, cluster_op_part_t *par
     );
     uint32_t object_offset = (part->op.req.rw.offset - op->offset) / pool_cfg.bitmap_granularity;
     uint32_t part_offset = (part->op.req.rw.offset % pg_block_size) / pool_cfg.bitmap_granularity;
-    uint32_t op_len = op->len / pool_cfg.bitmap_granularity;
-    uint32_t part_len = pg_block_size/pool_cfg.bitmap_granularity - part_offset;
-    if (part_len > op_len-object_offset)
-    {
-        part_len = op_len-object_offset;
-    }
+    uint32_t part_len = (op->opcode == OSD_OP_READ_BITMAP ? pg_block_size : part->op.req.rw.len) / pool_cfg.bitmap_granularity;
     if (!(object_offset & 0x7) && !(part_offset & 0x7) && (part_len >= 8))
     {
         // Copy bytes

src/cluster_client.h

@@ -11,7 +11,6 @@
 #define INODE_LIST_DONE 1
 #define INODE_LIST_HAS_UNSTABLE 2
 #define OSD_OP_READ_BITMAP OSD_OP_SEC_READ_BMP
-#define OSD_OP_READ_CHAIN_BITMAP 0x102
 
 #define OSD_OP_IGNORE_READONLY 0x08
 
@@ -31,7 +30,7 @@ struct cluster_op_part_t
 
 struct cluster_op_t
 {
-    uint64_t opcode; // OSD_OP_READ, OSD_OP_WRITE, OSD_OP_SYNC, OSD_OP_DELETE, OSD_OP_READ_BITMAP, OSD_OP_READ_CHAIN_BITMAP
+    uint64_t opcode; // OSD_OP_READ, OSD_OP_WRITE, OSD_OP_SYNC, OSD_OP_DELETE, OSD_OP_READ_BITMAP
     uint64_t inode;
     uint64_t offset;
     uint64_t len;
@@ -40,13 +39,9 @@ struct cluster_op_t
     uint64_t version = 0;
     // now only OSD_OP_IGNORE_READONLY is supported
     uint64_t flags = 0;
-    // negative retval is an error number
-    // write and read return len on success
-    // sync and delete return 0 on success
-    // read_bitmap and read_chain_bitmap return the length of bitmap in bits(!)
     int retval;
     osd_op_buf_list_t iov;
-    // READ, READ_BITMAP, READ_CHAIN_BITMAP return the bitmap here
+    // READ and READ_BITMAP return the bitmap here
     void *bitmap_buf = NULL;
     std::function<void(cluster_op_t*)> callback;
     ~cluster_op_t();
@@ -104,16 +99,12 @@ class cluster_client_t
     std::vector<std::function<void(void)>> on_ready_hooks;
     std::vector<inode_list_t*> lists;
     int continuing_ops = 0;
-    bool msgr_initialized = false;
 
 public:
     etcd_state_client_t st_cli;
-
     osd_messenger_t msgr;
-    void init_msgr();
-
-    json11::Json::object cli_config, file_config, etcd_global_config;
-    json11::Json::object config;
+    json11::Json config;
+    json11::Json::object merged_config;
 
     cluster_client_t(ring_loop_t *ringloop, timerfd_manager_t *tfd, json11::Json & config);
     ~cluster_client_t();

src/cluster_client_list.cpp

@@ -43,7 +43,6 @@ struct inode_list_t
 inode_list_t* cluster_client_t::list_inode_start(inode_t inode,
     std::function<void(inode_list_t* lst, std::set<object_id>&& objects, pg_num_t pg_num, osd_num_t primary_osd, int status)> callback)
 {
-    init_msgr();
     int skipped_pgs = 0;
     pool_id_t pool_id = INODE_POOL(inode);
     if (!pool_id || st_cli.pool_config.find(pool_id) == st_cli.pool_config.end())

src/disk_tool_journal.cpp

@@ -281,7 +281,7 @@ void disk_tool_t::dump_journal_entry(int num, journal_entry *je, bool json)
         if (je->big_write.size > sizeof(journal_entry_big_write))
         {
             printf(json ? ",\"bitmap\":\"" : " (bitmap: ");
-            for (int i = sizeof(journal_entry_big_write); i < je->big_write.size; i++)
+            for (int i = sizeof(journal_entry_big_write); i < je->small_write.size; i++)
             {
                 printf("%02x", ((uint8_t*)je)[i]);
             }

src/disk_tool_meta.cpp

@@ -26,7 +26,7 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
         buf_size = dsk.meta_len;
     void *data = memalign_or_die(MEM_ALIGNMENT, buf_size);
     lseek64(dsk.meta_fd, dsk.meta_offset, 0);
-    read_blocking(dsk.meta_fd, data, dsk.meta_block_size);
+    read_blocking(dsk.meta_fd, data, buf_size);
     // Check superblock
     blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
     if (hdr->zero == 0 &&
@@ -41,11 +41,8 @@ int disk_tool_t::process_meta(std::function<void(blockstore_meta_header_v1_t *)>
             if (buf_size % dsk.meta_block_size)
             {
                 buf_size = 8*dsk.meta_block_size;
-                void *new_data = memalign_or_die(MEM_ALIGNMENT, buf_size);
-                memcpy(new_data, data, dsk.meta_block_size);
                 free(data);
-                data = new_data;
-                hdr = (blockstore_meta_header_v1_t *)data;
+                data = memalign_or_die(MEM_ALIGNMENT, buf_size);
             }
         }
         dsk.bitmap_granularity = hdr->bitmap_granularity;

src/disk_tool_prepare.cpp

@@ -621,11 +621,6 @@ int disk_tool_t::prepare(std::vector<std::string> devices)
                 }
                 // Treat all disks as SSDs if not in the hybrid mode
                 prepare_one(options, hybrid && dev.is_hdd ? 1 : 0);
-                if (hybrid)
-                {
-                    options.erase("journal_device");
-                    options.erase("meta_device");
-                }
             }
         }
     }

src/disk_tool_utils.cpp

@@ -305,10 +305,10 @@ int write_zero(int fd, uint64_t offset, uint64_t size)
 json11::Json read_parttable(std::string dev)
 {
     std::string part_dump;
-    int r = shell_exec({ "sfdisk", "--json", dev }, "", &part_dump, NULL);
+    int r = shell_exec({ "sfdisk", "--dump", dev, "--json" }, "", &part_dump, NULL);
     if (r == 255)
     {
-        fprintf(stderr, "Error running sfdisk --json %s\n", dev.c_str());
+        fprintf(stderr, "Error running sfdisk --dump %s --json\n", dev.c_str());
         return json11::Json(false);
     }
     // Decode partition table
@@ -319,7 +319,7 @@ json11::Json read_parttable(std::string dev)
         pt = json11::Json::parse(part_dump, err);
         if (err != "")
         {
-            fprintf(stderr, "sfdisk --json %s returned bad JSON: %s\n", dev.c_str(), part_dump.c_str());
+            fprintf(stderr, "sfdisk --dump %s --json returned bad JSON: %s\n", dev.c_str(), part_dump.c_str());
             return json11::Json(false);
         }
         pt = pt["partitiontable"];

src/epoll_manager.cpp

@@ -54,13 +54,6 @@ void epoll_manager_t::set_fd_handler(int fd, bool wr, std::function<void(int, in
         ev.events = (wr ? EPOLLOUT : 0) | EPOLLIN | EPOLLRDHUP | EPOLLET;
         if (epoll_ctl(epoll_fd, exists ? EPOLL_CTL_MOD : EPOLL_CTL_ADD, fd, &ev) < 0)
         {
-            if (errno == ENOENT)
-            {
-                // The FD is probably already closed
-                epoll_ctl(epoll_fd, EPOLL_CTL_DEL, fd, NULL);
-                epoll_handlers.erase(fd);
-                return;
-            }
             throw std::runtime_error(std::string("epoll_ctl: ") + strerror(errno));
         }
         epoll_handlers[fd] = handler;

src/etcd_state_client.cpp

@@ -18,8 +18,12 @@ etcd_state_client_t::~etcd_state_client_t()
     }
     watches.clear();
     etcd_watches_initialised = -1;
+    if (ws_keepalive_timer >= 0)
+    {
+        tfd->clear_timer(ws_keepalive_timer);
+        ws_keepalive_timer = -1;
+    }
 #ifndef __MOCK__
-    stop_ws_keepalive();
     if (etcd_watch_ws)
     {
         http_close(etcd_watch_ws);
@@ -241,7 +245,6 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
         if (this->etcd_keepalive_timeout < 30)
             this->etcd_keepalive_timeout = 30;
     }
-    auto old_etcd_ws_keepalive_interval = this->etcd_ws_keepalive_interval;
     this->etcd_ws_keepalive_interval = config["etcd_ws_keepalive_interval"].uint64_value();
     if (this->etcd_ws_keepalive_interval <= 0)
     {
@@ -262,13 +265,6 @@ void etcd_state_client_t::parse_config(const json11::Json & config)
     {
         this->etcd_quick_timeout = 1000;
     }
-    if (this->etcd_ws_keepalive_interval != old_etcd_ws_keepalive_interval && ws_keepalive_timer >= 0)
-    {
-#ifndef __MOCK__
-        stop_ws_keepalive();
-        start_ws_keepalive();
-#endif
-    }
 }
 
 void etcd_state_client_t::pick_next_etcd()
@@ -482,20 +478,6 @@ void etcd_state_client_t::start_etcd_watcher()
     {
         on_start_watcher_hook(etcd_watch_ws);
     }
-    start_ws_keepalive();
-}
-
-void etcd_state_client_t::stop_ws_keepalive()
-{
-    if (ws_keepalive_timer >= 0)
-    {
-        tfd->clear_timer(ws_keepalive_timer);
-        ws_keepalive_timer = -1;
-    }
-}
-
-void etcd_state_client_t::start_ws_keepalive()
-{
     if (ws_keepalive_timer < 0)
     {
         ws_keepalive_timer = tfd->set_timer(etcd_ws_keepalive_interval*1000, true, [this](int)
@@ -885,6 +867,7 @@ void etcd_state_client_t::parse_state(const etcd_kv_t & kv)
         }
         else
         {
+            fprintf(stderr, "RECEIVED PG %u/%u HISTORY: %s\n", pool_id, pg_num, value.dump().c_str());
             auto & pg_cfg = this->pool_config[pool_id].pg_config[pg_num];
             pg_cfg.target_history.clear();
             pg_cfg.all_peers.clear();

src/etcd_state_client.h

@@ -132,8 +132,6 @@ public:
     void etcd_txn(json11::Json txn, int timeout, int retries, int interval, std::function<void(std::string, json11::Json)> callback);
     void etcd_txn_slow(json11::Json txn, std::function<void(std::string, json11::Json)> callback);
     void start_etcd_watcher();
-    void stop_ws_keepalive();
-    void start_ws_keepalive();
     void load_global_config();
     void load_pgs();
     void parse_state(const etcd_kv_t & kv);

src/messenger.cpp

@@ -157,10 +157,10 @@ void osd_messenger_t::parse_config(const json11::Json & config)
         this->rdma_max_sge = 128;
     this->rdma_max_send = config["rdma_max_send"].uint64_value();
     if (!this->rdma_max_send)
-        this->rdma_max_send = 8;
+        this->rdma_max_send = 1;
     this->rdma_max_recv = config["rdma_max_recv"].uint64_value();
     if (!this->rdma_max_recv)
-        this->rdma_max_recv = 16;
+        this->rdma_max_recv = 128;
     this->rdma_max_msg = config["rdma_max_msg"].uint64_value();
     if (!this->rdma_max_msg || this->rdma_max_msg > 128*1024*1024)
         this->rdma_max_msg = 129*1024;
@@ -534,9 +534,8 @@ bool osd_messenger_t::is_rdma_enabled()
 }
 #endif
 
-json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
+json11::Json osd_messenger_t::read_config(const json11::Json & config)
 {
-    json11::Json::object file_config;
     const char *config_path = config["config_path"].string_value() != ""
         ? config["config_path"].string_value().c_str() : VITASTOR_CONFIG_PATH;
     int fd = open(config_path, O_RDONLY);
@@ -544,14 +543,14 @@ json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
     {
         if (errno != ENOENT)
             fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
-        return file_config;
+        return config;
     }
     struct stat st;
     if (fstat(fd, &st) != 0)
     {
         fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
         close(fd);
-        return file_config;
+        return config;
     }
     std::string buf;
     buf.resize(st.st_size);
@@ -563,125 +562,23 @@ json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
         {
             fprintf(stderr, "Error reading %s: %s\n", config_path, strerror(errno));
             close(fd);
-            return file_config;
+            return config;
         }
         done += r;
     }
     close(fd);
     std::string json_err;
-    file_config = json11::Json::parse(buf, json_err).object_items();
+    json11::Json::object file_config = json11::Json::parse(buf, json_err).object_items();
     if (json_err != "")
     {
         fprintf(stderr, "Invalid JSON in %s: %s\n", config_path, json_err.c_str());
+        return config;
+    }
+    file_config.erase("config_path");
+    file_config.erase("osd_num");
+    for (auto kv: config.object_items())
+    {
+        file_config[kv.first] = kv.second;
     }
     return file_config;
 }
-
-static const char* cli_only_params[] = {
-    // The list has to be sorted
-    "bitmap_granularity",
-    "block_size",
-    "data_device",
-    "data_offset",
-    "data_size",
-    "disable_data_fsync",
-    "disable_device_lock",
-    "disable_journal_fsync",
-    "disable_meta_fsync",
-    "disk_alignment",
-    "flush_journal",
-    "immediate_commit",
-    "inmemory_journal",
-    "inmemory_metadata",
-    "journal_block_size",
-    "journal_device",
-    "journal_no_same_sector_overwrites",
-    "journal_offset",
-    "journal_sector_buffer_count",
-    "journal_size",
-    "meta_block_size",
-    "meta_buf_size",
-    "meta_device",
-    "meta_offset",
-    "osd_num",
-    "readonly",
-};
-
-static const char **cli_only_end = cli_only_params + (sizeof(cli_only_params)/sizeof(cli_only_params[0]));
-
-static const char* local_only_params[] = {
-    // The list has to be sorted
-    "config_path",
-    "rdma_device",
-    "rdma_gid_index",
-    "rdma_max_msg",
-    "rdma_max_recv",
-    "rdma_max_send",
-    "rdma_max_sge",
-    "rdma_mtu",
-    "rdma_port_num",
-    "tcp_header_buffer_size",
-    "use_rdma",
-    "use_sync_send_recv",
-};
-
-static const char **local_only_end = local_only_params + (sizeof(local_only_params)/sizeof(local_only_params[0]));
-
-// Basically could be replaced by std::lower_bound()...
-static int find_str_array(const char **start, const char **end, const std::string & s)
-{
-    int min = 0, max = end-start;
-    while (max-min >= 2)
-    {
-        int mid = (min+max)/2;
-        int r = strcmp(s.c_str(), start[mid]);
-        if (r < 0)
-            max = mid;
-        else if (r > 0)
-            min = mid;
-        else
-            return mid;
-    }
-    if (min < end-start && !strcmp(s.c_str(), start[min]))
-        return min;
-    return -1;
-}
-
-json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object & cli_config,
-    const json11::Json::object & file_config,
-    const json11::Json::object & etcd_global_config,
-    const json11::Json::object & etcd_osd_config)
-{
-    // Priority: most important -> less important:
-    // etcd_osd_config -> cli_config -> etcd_global_config -> file_config
-    json11::Json::object res = file_config;
-    for (auto & kv: file_config)
-    {
-        int cli_only = find_str_array(cli_only_params, cli_only_end, kv.first);
-        if (cli_only < 0)
-        {
-            res[kv.first] = kv.second;
-        }
-    }
-    for (auto & kv: etcd_global_config)
-    {
-        int local_only = find_str_array(local_only_params, local_only_end, kv.first);
-        if (local_only < 0)
-        {
-            res[kv.first] = kv.second;
-        }
-    }
-    for (auto & kv: cli_config)
-    {
-        res[kv.first] = kv.second;
-    }
-    for (auto & kv: etcd_osd_config)
-    {
-        int local_only = find_str_array(local_only_params, local_only_end, kv.first);
-        if (local_only < 0)
-        {
-            res[kv.first] = kv.second;
-        }
-    }
-    return res;
-}

src/messenger.h

@@ -138,7 +138,6 @@ protected:
 
     std::vector<int> read_ready_clients;
     std::vector<int> write_ready_clients;
-    // We don't use ringloop->set_immediate here because we may have no ringloop in client :)
     std::vector<std::function<void()>> set_immediate;
 
 public:
@@ -166,11 +165,7 @@ public:
     void accept_connections(int listen_fd);
     ~osd_messenger_t();
 
-    static json11::Json::object read_config(const json11::Json & config);
-    static json11::Json::object merge_configs(const json11::Json::object & cli_config,
-        const json11::Json::object & file_config,
-        const json11::Json::object & etcd_global_config,
-        const json11::Json::object & etcd_osd_config);
+    static json11::Json read_config(const json11::Json & config);
 
 #ifdef WITH_RDMA
     bool is_rdma_enabled();

src/mock/messenger.cpp

@@ -43,15 +43,7 @@ void osd_messenger_t::send_replies()
 {
 }
 
-json11::Json::object osd_messenger_t::read_config(const json11::Json & config)
+json11::Json osd_messenger_t::read_config(const json11::Json & config)
 {
-    return json11::Json::object();
-}
-
-json11::Json::object osd_messenger_t::merge_configs(const json11::Json::object & cli_config,
-    const json11::Json::object & file_config,
-    const json11::Json::object & etcd_global_config,
-    const json11::Json::object & etcd_osd_config)
-{
-    return cli_config;
+    return config;
 }

src/msgr_rdma.cpp

@@ -368,8 +368,9 @@ static void try_send_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
 bool osd_messenger_t::try_send_rdma(osd_client_t *cl)
 {
     auto rc = cl->rdma_conn;
-    if (!cl->send_list.size() || rc->cur_send >= rc->max_send)
+    if (!cl->send_list.size() || rc->cur_send > 0)
     {
+        // Only send one batch at a time
         return true;
     }
     uint64_t op_size = 0, op_sge = 0;
@@ -379,7 +380,6 @@ bool osd_messenger_t::try_send_rdma(osd_client_t *cl)
         iovec & iov = cl->send_list[rc->send_pos];
         if (op_size >= rc->max_msg || op_sge >= rc->max_sge)
         {
-            rc->send_sizes.push_back(op_size);
             try_send_rdma_wr(cl, sge, op_sge);
             op_sge = 0;
             op_size = 0;
@@ -405,24 +405,18 @@ bool osd_messenger_t::try_send_rdma(osd_client_t *cl)
     }
     if (op_sge > 0)
     {
-        rc->send_sizes.push_back(op_size);
         try_send_rdma_wr(cl, sge, op_sge);
     }
     return true;
 }
 
-static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
+static void try_recv_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge)
 {
-    ibv_sge sge = {
-        .addr = (uintptr_t)buf,
-        .length = (uint32_t)cl->rdma_conn->max_msg,
-        .lkey = cl->rdma_conn->ctx->mr->lkey,
-    };
     ibv_recv_wr *bad_wr = NULL;
     ibv_recv_wr wr = {
         .wr_id = (uint64_t)(cl->peer_fd*2),
-        .sg_list = &sge,
-        .num_sge = 1,
+        .sg_list = sge,
+        .num_sge = op_sge,
     };
     int err = ibv_post_recv(cl->rdma_conn->qp, &wr, &bad_wr);
     if (err || bad_wr)
@@ -440,7 +434,12 @@ bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
     {
         void *buf = malloc_or_die(rc->max_msg);
         rc->recv_buffers.push_back(buf);
-        try_recv_rdma_wr(cl, buf);
+        ibv_sge sge = {
+            .addr = (uintptr_t)buf,
+            .length = (uint32_t)rc->max_msg,
+            .lkey = rc->ctx->mr->lkey,
+        };
+        try_recv_rdma_wr(cl, &sge, 1);
     }
     return true;
 }
@@ -477,7 +476,6 @@ void osd_messenger_t::handle_rdma_events()
                 continue;
             }
             osd_client_t *cl = cl_it->second;
-            auto rc = cl->rdma_conn;
             if (wc[i].status != IBV_WC_SUCCESS)
             {
                 fprintf(stderr, "RDMA work request failed for client %d", client_id);
@@ -491,59 +489,44 @@ void osd_messenger_t::handle_rdma_events()
             }
             if (!is_send)
             {
-                rc->cur_recv--;
-                if (!handle_read_buffer(cl, rc->recv_buffers[rc->next_recv_buf], wc[i].byte_len))
+                cl->rdma_conn->cur_recv--;
+                if (!handle_read_buffer(cl, cl->rdma_conn->recv_buffers[0], wc[i].byte_len))
                 {
                     // handle_read_buffer may stop the client
                     continue;
                 }
-                try_recv_rdma_wr(cl, rc->recv_buffers[rc->next_recv_buf]);
-                rc->next_recv_buf = (rc->next_recv_buf+1) % rc->recv_buffers.size();
+                free(cl->rdma_conn->recv_buffers[0]);
+                cl->rdma_conn->recv_buffers.erase(cl->rdma_conn->recv_buffers.begin(), cl->rdma_conn->recv_buffers.begin()+1);
+                try_recv_rdma(cl);
             }
             else
             {
-                rc->cur_send--;
-                uint64_t sent_size = rc->send_sizes.at(0);
-                rc->send_sizes.erase(rc->send_sizes.begin(), rc->send_sizes.begin()+1);
-                int send_pos = 0, send_buf_pos = 0;
-                while (sent_size > 0)
+                cl->rdma_conn->cur_send--;
+                if (!cl->rdma_conn->cur_send)
                 {
-                    if (sent_size >= cl->send_list.at(send_pos).iov_len)
+                    // Wait for the whole batch
+                    for (int i = 0; i < cl->rdma_conn->send_pos; i++)
                     {
-                        sent_size -= cl->send_list[send_pos].iov_len;
-                        send_pos++;
+                        if (cl->outbox[i].flags & MSGR_SENDP_FREE)
+                        {
+                            // Reply fully sent
+                            delete cl->outbox[i].op;
+                        }
                     }
-                    else
+                    if (cl->rdma_conn->send_pos > 0)
                     {
-                        send_buf_pos = sent_size;
-                        sent_size = 0;
+                        cl->send_list.erase(cl->send_list.begin(), cl->send_list.begin()+cl->rdma_conn->send_pos);
+                        cl->outbox.erase(cl->outbox.begin(), cl->outbox.begin()+cl->rdma_conn->send_pos);
+                        cl->rdma_conn->send_pos = 0;
                     }
-                }
-                assert(rc->send_pos >= send_pos);
-                if (rc->send_pos == send_pos)
-                {
-                    rc->send_buf_pos -= send_buf_pos;
-                }
-                rc->send_pos -= send_pos;
-                for (int i = 0; i < send_pos; i++)
-                {
-                    if (cl->outbox[i].flags & MSGR_SENDP_FREE)
+                    if (cl->rdma_conn->send_buf_pos > 0)
                     {
-                        // Reply fully sent
-                        delete cl->outbox[i].op;
+                        cl->send_list[0].iov_base = (uint8_t*)cl->send_list[0].iov_base + cl->rdma_conn->send_buf_pos;
+                        cl->send_list[0].iov_len -= cl->rdma_conn->send_buf_pos;
+                        cl->rdma_conn->send_buf_pos = 0;
                     }
+                    try_send_rdma(cl);
                 }
-                if (send_pos > 0)
-                {
-                    cl->send_list.erase(cl->send_list.begin(), cl->send_list.begin()+send_pos);
-                    cl->outbox.erase(cl->outbox.begin(), cl->outbox.begin()+send_pos);
-                }
-                if (send_buf_pos > 0)
-                {
-                    cl->send_list[0].iov_base = (uint8_t*)cl->send_list[0].iov_base + send_buf_pos;
-                    cl->send_list[0].iov_len -= send_buf_pos;
-                }
-                try_send_rdma(cl);
             }
         }
     } while (event_count > 0);

src/msgr_rdma.h

@@ -49,9 +49,8 @@ struct msgr_rdma_connection_t
     uint64_t max_msg = 0;
 
     int send_pos = 0, send_buf_pos = 0;
-    int next_recv_buf = 0;
+    int recv_pos = 0, recv_buf_pos = 0;
     std::vector<void*> recv_buffers;
-    std::vector<uint64_t> send_sizes;
 
     ~msgr_rdma_connection_t();
     static msgr_rdma_connection_t *create(msgr_rdma_context_t *ctx, uint32_t max_send, uint32_t max_recv, uint32_t max_sge, uint32_t max_msg);

src/msgr_receive.cpp

@@ -103,10 +103,7 @@ bool osd_messenger_t::handle_read(int result, osd_client_t *cl)
             cl->recv_list.eat(result);
             if (cl->recv_list.done >= cl->recv_list.count)
             {
-                if (!handle_finished_read(cl))
-                {
-                    goto fin;
-                }
+                handle_finished_read(cl);
             }
         }
         if (result >= cl->read_iov.iov_len)
@@ -316,18 +313,17 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
             stop_client(cl->peer_fd);
             return false;
         }
-        if (bmp_len > 0)
+        if (op->reply.hdr.retval >= 0 && bmp_len > 0)
         {
             assert(op->bitmap);
             cl->recv_list.push_back(op->bitmap, bmp_len);
-            cl->read_remaining += bmp_len;
         }
         if (op->reply.hdr.retval > 0)
         {
             assert(op->iov.count > 0);
             cl->recv_list.append(op->iov);
-            cl->read_remaining += op->reply.hdr.retval;
         }
+        cl->read_remaining = op->reply.hdr.retval + bmp_len;
         if (cl->read_remaining == 0)
         {
             goto reuse;

src/object_id.h

@@ -39,11 +39,6 @@ struct __attribute__((__packed__)) obj_ver_id
     uint64_t version;
 };
 
-inline bool operator == (const obj_ver_id & a, const obj_ver_id & b)
-{
-    return a.oid == b.oid && a.version == b.version;
-}
-
 inline bool operator < (const obj_ver_id & a, const obj_ver_id & b)
 {
     return a.oid < b.oid || a.oid == b.oid && a.version < b.version;

src/osd.cpp

@@ -35,18 +35,18 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
 
     this->ringloop = ringloop;
 
-    this->cli_config = config.object_items();
-    this->file_config = msgr.read_config(this->cli_config);
-    parse_config(true);
+    this->config = msgr.read_config(config).object_items();
+    if (this->config.find("log_level") == this->config.end())
+        this->config["log_level"] = 1;
+    parse_config(this->config, true);
 
     epmgr = new epoll_manager_t(ringloop);
     // FIXME: Use timerfd_interval based directly on io_uring
     this->tfd = epmgr->tfd;
 
-    if (!json_is_true(this->config["disable_blockstore"]))
+    auto bs_cfg = json_to_bs(this->config);
+    this->bs = new blockstore_t(bs_cfg, ringloop, tfd);
     {
-        auto bs_cfg = json_to_bs(this->config);
-        this->bs = new blockstore_t(bs_cfg, ringloop, tfd);
         // Autosync based on the number of unstable writes to prevent stalls due to insufficient journal space
         uint64_t max_autosync = bs->get_journal_size() / bs->get_block_size() / 2;
         if (autosync_writes > max_autosync)
@@ -67,11 +67,11 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
         }
     }
 
-    print_stats_timer_id = this->tfd->set_timer(print_stats_interval*1000, true, [this](int timer_id)
+    this->tfd->set_timer(print_stats_interval*1000, true, [this](int timer_id)
     {
         print_stats();
     });
-    slow_log_timer_id = this->tfd->set_timer(slow_log_interval*1000, true, [this](int timer_id)
+    this->tfd->set_timer(slow_log_interval*1000, true, [this](int timer_id)
     {
         print_slow();
     });
@@ -91,42 +91,18 @@ osd_t::osd_t(const json11::Json & config, ring_loop_t *ringloop)
 
 osd_t::~osd_t()
 {
-    if (slow_log_timer_id >= 0)
-    {
-        tfd->clear_timer(slow_log_timer_id);
-        slow_log_timer_id = -1;
-    }
-    if (print_stats_timer_id >= 0)
-    {
-        tfd->clear_timer(print_stats_timer_id);
-        print_stats_timer_id = -1;
-    }
-    if (autosync_timer_id >= 0)
-    {
-        tfd->clear_timer(autosync_timer_id);
-        autosync_timer_id = -1;
-    }
     ringloop->unregister_consumer(&consumer);
     delete epmgr;
-    if (bs)
-        delete bs;
+    delete bs;
     close(listen_fd);
     free(zero_buffer);
 }
 
-void osd_t::parse_config(bool init)
+void osd_t::parse_config(const json11::Json & config, bool allow_disk_params)
 {
-    config = msgr.merge_configs(cli_config, file_config, etcd_global_config, etcd_osd_config);
-    if (config.find("log_level") == this->config.end())
-        config["log_level"] = 1;
-    if (bs)
-    {
-        auto bs_cfg = json_to_bs(config);
-        bs->parse_config(bs_cfg);
-    }
     st_cli.parse_config(config);
     msgr.parse_config(config);
-    if (init)
+    if (allow_disk_params)
     {
         // OSD number
         osd_num = config["osd_num"].uint64_value();
@@ -148,27 +124,24 @@ void osd_t::parse_config(bool init)
             immediate_commit = IMMEDIATE_SMALL;
         else
             immediate_commit = IMMEDIATE_NONE;
-        // Bind address
-        bind_address = config["bind_address"].string_value();
-        if (bind_address == "")
-            bind_address = "0.0.0.0";
-        bind_port = config["bind_port"].uint64_value();
-        if (bind_port <= 0 || bind_port > 65535)
-            bind_port = 0;
-        // OSD configuration
-        etcd_report_interval = config["etcd_report_interval"].uint64_value();
-        if (etcd_report_interval <= 0)
-            etcd_report_interval = 5;
-        readonly = json_is_true(config["readonly"]);
-        run_primary = !json_is_false(config["run_primary"]);
-        allow_test_ops = json_is_true(config["allow_test_ops"]);
     }
+    // Bind address
+    bind_address = config["bind_address"].string_value();
+    if (bind_address == "")
+        bind_address = "0.0.0.0";
+    bind_port = config["bind_port"].uint64_value();
+    if (bind_port <= 0 || bind_port > 65535)
+        bind_port = 0;
+    // OSD configuration
     log_level = config["log_level"].uint64_value();
-    auto old_no_rebalance = no_rebalance;
+    etcd_report_interval = config["etcd_report_interval"].uint64_value();
+    if (etcd_report_interval <= 0)
+        etcd_report_interval = 5;
+    readonly = json_is_true(config["readonly"]);
+    run_primary = !json_is_false(config["run_primary"]);
     no_rebalance = json_is_true(config["no_rebalance"]);
-    auto old_no_recovery = no_recovery;
     no_recovery = json_is_true(config["no_recovery"]);
-    auto old_autosync_interval = autosync_interval;
+    allow_test_ops = json_is_true(config["allow_test_ops"]);
     if (!config["autosync_interval"].is_null())
     {
         // Allow to set it to 0
@@ -196,46 +169,15 @@ void osd_t::parse_config(bool init)
     recovery_sync_batch = config["recovery_sync_batch"].uint64_value();
     if (recovery_sync_batch < 1 || recovery_sync_batch > MAX_RECOVERY_QUEUE)
         recovery_sync_batch = DEFAULT_RECOVERY_BATCH;
-    auto old_print_stats_interval = print_stats_interval;
     print_stats_interval = config["print_stats_interval"].uint64_value();
     if (!print_stats_interval)
         print_stats_interval = 3;
-    auto old_slow_log_interval = slow_log_interval;
     slow_log_interval = config["slow_log_interval"].uint64_value();
     if (!slow_log_interval)
         slow_log_interval = 10;
     inode_vanish_time = config["inode_vanish_time"].uint64_value();
     if (!inode_vanish_time)
         inode_vanish_time = 60;
-    if ((old_no_rebalance && !no_rebalance || old_no_recovery && !no_recovery) &&
-        !(peering_state & (OSD_RECOVERING | OSD_FLUSHING_PGS)))
-    {
-        peering_state = peering_state | OSD_RECOVERING;
-    }
-    if (old_autosync_interval != autosync_interval && autosync_timer_id >= 0)
-    {
-        this->tfd->clear_timer(autosync_timer_id);
-        autosync_timer_id = this->tfd->set_timer(autosync_interval*1000, true, [this](int timer_id)
-        {
-            autosync();
-        });
-    }
-    if (old_print_stats_interval != print_stats_interval && print_stats_timer_id >= 0)
-    {
-        tfd->clear_timer(print_stats_timer_id);
-        print_stats_timer_id = this->tfd->set_timer(print_stats_interval*1000, true, [this](int timer_id)
-        {
-            print_stats();
-        });
-    }
-    if (old_slow_log_interval != slow_log_interval && slow_log_timer_id >= 0)
-    {
-        tfd->clear_timer(slow_log_timer_id);
-        slow_log_timer_id = this->tfd->set_timer(slow_log_interval*1000, true, [this](int timer_id)
-        {
-            print_slow();
-        });
-    }
 }
 
 void osd_t::bind_socket()
@@ -460,7 +402,7 @@ void osd_t::print_slow()
                 int l = sizeof(alloc), n;
                 char *buf = alloc;
 #define bufprintf(s, ...) { n = snprintf(buf, l, s, __VA_ARGS__); n = n < 0 ? 0 : n; buf += n; l -= n; }
-                bufprintf("[OSD %lu] Slow op %lx", osd_num, (unsigned long)op);
+                bufprintf("[OSD %lu] Slow op", osd_num);
                 if (kv.second->osd_num)
                 {
                     bufprintf(" from peer OSD %lu (client %d)", kv.second->osd_num, kv.second->peer_fd);
@@ -533,7 +475,7 @@ void osd_t::print_slow()
             }
         }
     }
-    if (has_slow && bs)
+    if (has_slow)
     {
         bs->dump_diagnostics();
     }

src/osd.h

@@ -90,7 +90,7 @@ class osd_t
 {
     // config
 
-    json11::Json::object cli_config, file_config, etcd_global_config, etcd_osd_config, config;
+    json11::Json::object config;
     int etcd_report_interval = 5;
 
     bool readonly = false;
@@ -126,7 +126,6 @@ class osd_t
     bool pg_config_applied = false;
     bool etcd_reporting_pg_state = false;
     bool etcd_reporting_stats = false;
-    int autosync_timer_id = -1, print_stats_timer_id = -1, slow_log_timer_id = -1;
 
     // peers and PGs
 
@@ -153,7 +152,7 @@ class osd_t
 
     bool stopping = false;
     int inflight_ops = 0;
-    blockstore_t *bs = NULL;
+    blockstore_t *bs;
     void *zero_buffer = NULL;
     uint64_t zero_buffer_size = 0;
     uint32_t bs_block_size, bs_bitmap_granularity, clean_entry_bitmap_size;
@@ -174,7 +173,7 @@ class osd_t
     uint64_t recovery_stat_bytes[2][2] = {};
 
     // cluster connection
-    void parse_config(bool init);
+    void parse_config(const json11::Json & config, bool allow_disk_params);
     void init_cluster();
     void on_change_osd_state_hook(osd_num_t peer_osd);
     void on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num);

src/osd_cluster.cpp

@@ -75,7 +75,7 @@ void osd_t::init_cluster()
     }
     if (run_primary && autosync_interval > 0)
     {
-        autosync_timer_id = this->tfd->set_timer(autosync_interval*1000, true, [this](int timer_id)
+        this->tfd->set_timer(autosync_interval*1000, true, [this](int timer_id)
         {
             autosync();
         });
@@ -182,10 +182,10 @@ json11::Json osd_t::get_statistics()
     char time_str[50] = { 0 };
     sprintf(time_str, "%ld.%03ld", ts.tv_sec, ts.tv_nsec/1000000);
     st["time"] = time_str;
+    st["blockstore_ready"] = bs->is_started();
+    st["data_block_size"] = (uint64_t)bs->get_block_size();
     if (bs)
     {
-        st["blockstore_ready"] = bs->is_started();
-        st["data_block_size"] = (uint64_t)bs->get_block_size();
         st["size"] = bs->get_block_count() * bs->get_block_size();
         st["free"] = bs->get_free_block_count() * bs->get_block_size();
     }
@@ -233,8 +233,7 @@ void osd_t::report_statistics()
     json11::Json::object inode_space;
     json11::Json::object last_stat;
     pool_id_t last_pool = 0;
-    std::map<uint64_t, uint64_t> bs_empty_space;
-    auto & bs_inode_space = bs ? bs->get_inode_space_stats() : bs_empty_space;
+    auto & bs_inode_space = bs->get_inode_space_stats();
     for (auto kv: bs_inode_space)
     {
         pool_id_t pool_id = INODE_POOL(kv.first);
@@ -375,11 +374,7 @@ void osd_t::on_change_osd_state_hook(osd_num_t peer_osd)
 
 void osd_t::on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes)
 {
-    if (changes.find(st_cli.etcd_prefix+"/config/global") != changes.end())
-    {
-        etcd_global_config = changes[st_cli.etcd_prefix+"/config/global"].value.object_items();
-        parse_config(false);
-    }
+    // FIXME apply config changes in runtime (maybe, some)
     if (run_primary)
     {
         apply_pg_count();
@@ -389,8 +384,11 @@ void osd_t::on_change_etcd_state_hook(std::map<std::string, etcd_kv_t> & changes
 
 void osd_t::on_load_config_hook(json11::Json::object & global_config)
 {
-    etcd_global_config = global_config;
-    parse_config(true);
+    json11::Json::object osd_config = this->config;
+    for (auto & kv: global_config)
+        if (osd_config.find(kv.first) == osd_config.end())
+            osd_config[kv.first] = kv.second;
+    parse_config(osd_config, false);
     bind_socket();
     acquire_lease();
 }
@@ -685,7 +683,7 @@ void osd_t::apply_pg_config()
                 auto vec_all_peers = std::vector<osd_num_t>(all_peers.begin(), all_peers.end());
                 if (currently_taken)
                 {
-                    if (pg_it->second.state & (PG_ACTIVE | PG_INCOMPLETE | PG_PEERING | PG_REPEERING))
+                    if (pg_it->second.state & (PG_ACTIVE | PG_INCOMPLETE | PG_PEERING | PG_REPEERING | PG_PEERED))
                     {
                         if (pg_it->second.target_set == pg_cfg.target_set &&
                             pg_it->second.target_history == pg_cfg.target_history &&
@@ -696,6 +694,20 @@ void osd_t::apply_pg_config()
                         }
                         else
                         {
+                            printf(
+                                "Repeer %u/%u because of history: %s vs %s\n",
+                                pool_id, pg_num,
+                                json11::Json(json11::Json::object {
+                                    { "target_set", pg_cfg.target_set },
+                                    { "osd_sets", pg_cfg.target_history },
+                                    { "all_peers", vec_all_peers },
+                                }).dump().c_str(),
+                                json11::Json(json11::Json::object {
+                                    { "target_set", pg_it->second.target_set },
+                                    { "osd_sets", pg_it->second.target_history },
+                                    { "all_peers", pg_it->second.all_peers },
+                                }).dump().c_str()
+                            );
                             // Stop PG, reapply change after stopping
                             stop_pg(pg_it->second);
                             all_applied = false;
@@ -736,7 +748,7 @@ void osd_t::apply_pg_config()
                     .pg_cursize = 0,
                     .pg_size = pool_item.second.pg_size,
                     .pg_minsize = pool_item.second.pg_minsize,
-                    .pg_data_size = pool_item.second.scheme == POOL_SCHEME_REPLICATED
+                    .pg_data_size = pg.scheme == POOL_SCHEME_REPLICATED
                          ? 1 : pool_item.second.pg_size - pool_item.second.parity_chunks,
                     .pool_id = pool_id,
                     .pg_num = pg_num,
@@ -812,21 +824,11 @@ void osd_t::report_pg_states()
                     pg_it->second.cur_state != 0)
                 {
                     pg_state_exists = true;
-                    if (pg.state == PG_OFFLINE && pg_it->second.cur_primary != this->osd_num)
-                    {
-                        // Nothing to check or report, PG is already taken over by another OSD
-                        continue;
-                    }
                 }
             }
         }
         if (!pg_state_exists)
         {
-            if (pg.state == PG_OFFLINE)
-            {
-                // Nothing to check or report, PG is already stopped
-                continue;
-            }
             // Check that the PG key does not exist
             // Failed check indicates an unsuccessful PG lock attempt in this case
             checks.push_back(json11::Json::object {
@@ -883,6 +885,7 @@ void osd_t::report_pg_states()
                     { "all_peers", pg.all_peers },
                     { "osd_sets", pg.target_history },
                 };
+                printf("PG %u/%u HISTORY -> %s\n", pg.pool_id, pg.pg_num, json11::Json(history_value).dump().c_str());
                 checks.push_back(json11::Json::object {
                     { "target", "MOD" },
                     { "key", history_key },
@@ -911,15 +914,6 @@ void osd_t::report_pg_states()
     {
         etcd_reporting_pg_state = false;
         if (!data["succeeded"].bool_value())
-        {
-            std::string rpgnames = "";
-            for (auto pp: reporting_pgs)
-            {
-                rpgnames += (rpgnames.size() ? ", " : "")+std::to_string(pp.pool_pg_num.pool_id)+"/"+std::to_string(pp.pool_pg_num.pg_num);
-            }
-            printf("Error reporting PG %s states, will repeat the attempt: %s\n", rpgnames.c_str(), err.c_str());
-        }
-        if (!data["succeeded"].bool_value())
         {
             // One of PG state updates failed, put dirty flags back
             for (auto pp: reporting_pgs)
@@ -984,6 +978,13 @@ void osd_t::report_pg_states()
                         }
                         this->pgs.erase(pg_it);
                     }
+                    else if (pg_it->second.state & PG_PEERED)
+                    {
+                        // Activate PG after PG PEERED state is reported along with history
+                        // (if the state wasn't changed again)
+                        pg_it->second.state = pg_it->second.state & ~PG_PEERED | PG_ACTIVE;
+                        report_pg_state(pg_it->second);
+                    }
                 }
             }
             // Push other PG state updates, if any

src/osd_flush.cpp

@@ -64,11 +64,6 @@ void osd_t::submit_pg_flush_ops(pg_t & pg)
 
 void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval)
 {
-    if (log_level > 2)
-    {
-        printf("[PG %u/%u] flush batch %lx completed on OSD %lu with result %d\n",
-            pool_id, pg_num, (uint64_t)fb, peer_osd, retval);
-    }
     pool_pg_num_t pg_id = { .pool_id = pool_id, .pg_num = pg_num };
     if (pgs.find(pg_id) == pgs.end() || pgs[pg_id].flush_batch != fb)
     {
@@ -104,9 +99,10 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
         std::vector<osd_op_t*> continue_ops;
         auto & pg = pgs.at(pg_id);
         auto it = pg.flush_actions.begin(), prev_it = it;
+        auto erase_start = it;
         while (1)
         {
-            if (it == pg.flush_actions.end() || !it->second.submitted ||
+            if (it == pg.flush_actions.end() ||
                 it->first.oid.inode != prev_it->first.oid.inode ||
                 (it->first.oid.stripe & ~STRIPE_MASK) != (prev_it->first.oid.stripe & ~STRIPE_MASK))
             {
@@ -120,23 +116,29 @@ void osd_t::handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, p
                 });
                 if (wr_it != pg.write_queue.end())
                 {
-                    if (log_level > 2)
-                    {
-                        printf("[PG %u/%u] continuing write %lx to object %lx:%lx after flush\n",
-                            pool_id, pg_num, (uint64_t)wr_it->second, wr_it->first.inode, wr_it->first.stripe);
-                    }
                     continue_ops.push_back(wr_it->second);
+                    pg.write_queue.erase(wr_it);
                 }
             }
-            if (it == pg.flush_actions.end() || !it->second.submitted)
+            if ((it == pg.flush_actions.end() || !it->second.submitted) &&
+                erase_start != it)
+            {
+                pg.flush_actions.erase(erase_start, it);
+            }
+            if (it == pg.flush_actions.end())
             {
-                if (it != pg.flush_actions.begin())
-                {
-                    pg.flush_actions.erase(pg.flush_actions.begin(), it);
-                }
                 break;
             }
-            prev_it = it++;
+            prev_it = it;
+            if (!it->second.submitted)
+            {
+                it++;
+                erase_start = it;
+            }
+            else
+            {
+                it++;
+            }
         }
         delete fb;
         pg.flush_batch = NULL;
@@ -166,18 +168,6 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
     // Copy buffer so it gets freed along with the operation
     op->buf = malloc_or_die(sizeof(obj_ver_id) * count);
     memcpy(op->buf, data, sizeof(obj_ver_id) * count);
-    if (log_level > 2)
-    {
-        printf(
-            "[PG %u/%u] flush batch %lx on OSD %lu: %s objects: ",
-            pool_id, pg_num, (uint64_t)fb, peer_osd, rollback ? "rollback" : "stabilize"
-        );
-        for (int i = 0; i < count; i++)
-        {
-            printf(i > 0 ? ", %lx:%lx v%lu" : "%lx:%lx v%lu", data[i].oid.inode, data[i].oid.stripe, data[i].version);
-        }
-        printf("\n");
-    }
     if (peer_osd == this->osd_num)
     {
         // local
@@ -314,10 +304,9 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
             {
                 // PG is stopped or one of the OSDs is gone, error is harmless
                 printf(
-                    "[PG %u/%u] Recovery operation failed with object %lx:%lx\n",
-                    INODE_POOL(op->oid.inode),
-                    map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size),
-                    op->oid.inode, op->oid.stripe
+                    "Recovery operation failed with object %lx:%lx (PG %u/%u)\n",
+                    op->oid.inode, op->oid.stripe, INODE_POOL(op->oid.inode),
+                    map_to_pg(op->oid, st_cli.pool_config.at(INODE_POOL(op->oid.inode)).pg_stripe_size)
                 );
             }
             else

src/osd_ops.h

@@ -191,7 +191,7 @@ struct __attribute__((__packed__)) osd_op_rw_t
     uint64_t inode;
     // offset
     uint64_t offset;
-    // length. 0 means to read all bitmaps of the specified range, but no data.
+    // length
     uint32_t len;
     // flags (for future)
     uint32_t flags;

src/osd_peering.cpp

@@ -50,6 +50,10 @@ void osd_t::handle_peers()
                     still = true;
                 }
             }
+            else if (p.second.state & PG_PEERED)
+            {
+                still = true;
+            }
         }
         if (!still)
         {
@@ -70,13 +74,17 @@ void osd_t::handle_peers()
                 }
                 still = true;
             }
+            else if (p.second.state & PG_PEERED)
+            {
+                still = true;
+            }
         }
         if (!still)
         {
             peering_state = peering_state & ~OSD_FLUSHING_PGS | OSD_RECOVERING;
         }
     }
-    if (!(peering_state & OSD_FLUSHING_PGS) && (peering_state & OSD_RECOVERING) && !readonly)
+    if ((peering_state & OSD_RECOVERING) && !readonly)
     {
         if (!continue_recovery())
         {
@@ -92,7 +100,7 @@ void osd_t::repeer_pgs(osd_num_t peer_osd)
     {
         auto & pg = p.second;
         bool repeer = false;
-        if (pg.state & (PG_PEERING | PG_ACTIVE | PG_INCOMPLETE))
+        if (pg.state & (PG_PEERING | PG_PEERED | PG_ACTIVE | PG_INCOMPLETE))
         {
             for (osd_num_t pg_osd: pg.all_peers)
             {
@@ -483,10 +491,6 @@ void osd_t::report_pg_state(pg_t & pg)
         pg.all_peers = pg.target_set;
         std::sort(pg.all_peers.begin(), pg.all_peers.end());
         pg.cur_peers = pg.target_set;
-        // Change pg_config at the same time, otherwise our PG reconciling loop may try to apply the old metadata
-        auto & pg_cfg = st_cli.pool_config[pg.pool_id].pg_config[pg.pg_num];
-        pg_cfg.target_history = pg.target_history;
-        pg_cfg.all_peers = pg.all_peers;
     }
     else if (pg.state == (PG_ACTIVE|PG_LEFT_ON_DEAD))
     {
@@ -526,9 +530,6 @@ void osd_t::report_pg_state(pg_t & pg)
                 pg.cur_peers.push_back(pg_osd);
             }
         }
-        auto & pg_cfg = st_cli.pool_config[pg.pool_id].pg_config[pg.pg_num];
-        pg_cfg.target_history = pg.target_history;
-        pg_cfg.all_peers = pg.all_peers;
     }
     if (pg.state == PG_OFFLINE && !this->pg_config_applied)
     {

src/osd_peering_pg.cpp

@@ -88,10 +88,14 @@ void pg_obj_state_check_t::walk()
     {
         // Activate as degraded
         // Current OSD set will be added into target_history on first write
-        pg->state |= PG_DEGRADED;
+        pg->state |= PG_DEGRADED | PG_PEERED;
     }
-    pg->state |= PG_ACTIVE;
-    if (pg->cur_peers.size() < pg->all_peers.size())
+    else
+    {
+        // Just activate
+        pg->state |= PG_ACTIVE;
+    }
+    if (pg->state == PG_ACTIVE && pg->cur_peers.size() < pg->all_peers.size())
     {
         pg->state |= PG_LEFT_ON_DEAD;
     }
@@ -456,10 +460,11 @@ void pg_t::calc_object_states(int log_level)
 void pg_t::print_state()
 {
     printf(
-        "[PG %u/%u] is %s%s%s%s%s%s%s%s%s%s%s%s%s%s (%lu objects)\n", pool_id, pg_num,
+        "[PG %u/%u] is %s%s%s%s%s%s%s%s%s%s%s%s%s%s%s (%lu objects)\n", pool_id, pg_num,
         (state & PG_STARTING) ? "starting" : "",
         (state & PG_OFFLINE) ? "offline" : "",
         (state & PG_PEERING) ? "peering" : "",
+        (state & PG_PEERED) ? "peered" : "",
         (state & PG_INCOMPLETE) ? "incomplete" : "",
         (state & PG_ACTIVE) ? "active" : "",
         (state & PG_REPEERING) ? "repeering" : "",