Merge pull request 'master' (#3) from vitalif/vitastor:master into master

Reviewed-on: #3

.dockerignore

@@ -0,0 +1,19 @@
+.git
+build
+packages
+mon/node_modules
+*.o
+*.so
+osd
+stub_osd
+stub_uring_osd
+stub_bench
+osd_test
+dump_journal
+nbd_proxy
+rm_inode
+fio
+qemu
+rpm/*.Dockerfile
+debian/*.Dockerfile
+Dockerfile

.gitea/workflows/buildenv.Dockerfile

@@ -0,0 +1,36 @@
+FROM node:16-bullseye
+
+WORKDIR /root
+
+ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d
+
+RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \
+    echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \
+    echo >> /etc/apt/preferences; \
+    echo 'Package: *' >> /etc/apt/preferences; \
+    echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \
+    echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    echo >> /etc/apt/preferences; \
+    echo 'Package: *' >> /etc/apt/preferences; \
+    echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \
+    echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update
+RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
+    liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
+RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
+RUN apt-get -y install jq lp-solve sudo
+RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
+
+RUN set -ex; \
+    mkdir qemu-build; \
+    cd qemu-build; \
+    dpkg-source -x /root/qemu*.dsc; \
+    cd qemu*/; \
+    debian/rules configure-qemu || debian/rules b/configure-stamp; \
+    cd b/qemu; \
+    make -j8 config-poison.h || true; \
+    make -j8 qapi/qapi-builtin-types.h

.gitea/workflows/test.Dockerfile

@@ -0,0 +1,19 @@
+FROM git.yourcmc.ru/vitalif/vitastor/buildenv
+
+ADD . /root/vitastor
+
+RUN set -e -x; \
+    mkdir -p /root/fio-build/; \
+    cd /root/fio-build/; \
+    dpkg-source -x /root/fio*.dsc; \
+    cd /root/vitastor; \
+    ln -s /root/fio-build/fio-*/ ./fio; \
+    ln -s /root/qemu-build/qemu-*/ ./qemu; \
+    ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
+    cd mon; \
+    npm install; \
+    cd ..; \
+    mkdir build; \
+    cd build; \
+    cmake .. -DWITH_ASAN=yes -DWITH_QEMU=yes; \
+    make -j16

.gitea/workflows/test.yml

@@ -0,0 +1,858 @@
+name: Test
+
+on:
+  push:
+    branches:
+    - '*'
+    paths:
+    - '.gitea/**'
+    - 'src/**'
+    - 'mon/**'
+    - 'json11'
+    - 'cpp-btree'
+    - 'tests/**'
+
+env:
+  BUILDENV_IMAGE: git.yourcmc.ru/vitalif/vitastor/buildenv
+  TEST_IMAGE: git.yourcmc.ru/vitalif/vitastor/test
+  OSD_ARGS: '--etcd_quick_timeout 2000'
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+
+  buildenv:
+    runs-on: ubuntu-latest
+    container: git.yourcmc.ru/vitalif/gitea-ci-dind
+    steps:
+    - uses: actions/checkout@v3
+
+    - name: Build and push
+      run: |
+        set -ex
+        if ! docker manifest inspect $BUILDENV_IMAGE >/dev/null; then
+          docker build -t $BUILDENV_IMAGE -f .gitea/workflows/buildenv.Dockerfile .
+          docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}"
+          docker push $BUILDENV_IMAGE
+        fi
+
+  build:
+    runs-on: ubuntu-latest
+    needs: buildenv
+    container: git.yourcmc.ru/vitalif/gitea-ci-dind
+    steps:
+    - uses: actions/checkout@v3
+      with:
+        submodules: true
+
+    - name: Build and push
+      run: |
+        set -ex
+        if ! docker manifest inspect $TEST_IMAGE:$GITHUB_SHA >/dev/null; then
+          docker build -t $TEST_IMAGE:$GITHUB_SHA -f .gitea/workflows/test.Dockerfile .
+          docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}"
+          docker push $TEST_IMAGE:$GITHUB_SHA
+        fi
+
+  make_test:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    # leak sanitizer sometimes crashes
+    - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test
+
+  test_add_osd:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: /root/vitastor/tests/test_add_osd.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_cas:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_cas.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_change_pg_count:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_change_pg_count.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_change_pg_count_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_change_pg_count.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_change_pg_size:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_change_pg_size.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_create_nomaxid:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_create_nomaxid.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_etcd_fail:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: /root/vitastor/tests/test_etcd_fail.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_interrupted_rebalance:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_interrupted_rebalance_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_interrupted_rebalance_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_interrupted_rebalance_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_failure_domain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_failure_domain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_minsize_1:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_minsize_1.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_move_reappear:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_move_reappear.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_rm.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_chain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_chain_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 6
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot_chain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_down:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_snapshot_down_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_snapshot_down.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_splitbrain:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_splitbrain.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_rebalance_verify_ec_imm:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_switch_primary:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_switch_primary.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_write:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_write_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=xor /root/vitastor/tests/test_write.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_write_no_same:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_write_no_same.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_pg_size_2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: PG_SIZE=2 /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: SCHEME=ec /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_32k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k_dmj OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_32k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k_dj  OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_32k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_32k     OSD_ARGS="--data_csum_type crc32c --csum_block_size 32k" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k_dmj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata false --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k_dj:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_heal_csum_4k:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 10
+      run: TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS /root/vitastor/tests/test_heal.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub_zero_osd_2:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: ZERO_OSD=2 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub_xor:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=xor /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub_pg_size_3:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: PG_SIZE=3 /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_scrub_ec:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: SCHEME=ec /root/vitastor/tests/test_scrub.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+

.gitea/workflows/tests-to-yaml.pl

@@ -0,0 +1,79 @@
+#!/usr/bin/perl
+
+use strict;
+
+for my $line (<>)
+{
+    if ($line =~ /\.\/(test_[^\.]+)/s)
+    {
+        chomp $line;
+        my $base_name = $1;
+        my $test_name = $base_name;
+        my $timeout = 3;
+        if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_add_osd' ||
+            $test_name eq 'test_interrupted_rebalance' || $test_name eq 'test_rebalance_verify')
+        {
+            $timeout = 10;
+        }
+        while ($line =~ /([^\s=]+)=(\S+)/gs)
+        {
+            if ($1 eq 'TEST_NAME')
+            {
+                $test_name = $base_name.'_'.$2;
+                last;
+            }
+            elsif ($1 eq 'SCHEME' && $2 eq 'ec')
+            {
+                $test_name .= '_ec';
+            }
+            elsif ($1 eq 'SCHEME' && $2 eq 'xor')
+            {
+                $test_name .= '_xor';
+            }
+            elsif ($1 eq 'IMMEDIATE_COMMIT')
+            {
+                $test_name .= '_imm';
+            }
+            else
+            {
+                $test_name .= '_'.lc($1).'_'.$2;
+            }
+        }
+        if ($test_name eq 'test_snapshot_chain_ec')
+        {
+            $timeout = 6;
+        }
+        $line =~ s!\./test_!/root/vitastor/tests/test_!;
+        # Gitea CI doesn't support artifacts yet, lol
+        #- name: Upload results
+        #  uses: actions/upload-artifact\@v3
+        #  if: always()
+        #  with:
+        #    name: ${test_name}_result
+        #    path: |
+        #      /root/vitastor/testdata
+        #      !/root/vitastor/testdata/*.bin
+        #    retention-days: 5
+        print <<"EOF"
+  $test_name:
+    runs-on: ubuntu-latest
+    needs: build
+    container: \${{env.TEST_IMAGE}}:\${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: $timeout
+      run: $line
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- \$i --------"
+          cat \$i
+          echo ""
+        done
+
+EOF
+;
+    }
+}

.gitignore

@@ -0,0 +1,18 @@
+*.o
+*.so
+package-lock.json
+fio
+qemu
+osd
+stub_osd
+stub_uring_osd
+stub_bench
+osd_test
+osd_peering_pg_test
+dump_journal
+nbd_proxy
+rm_inode
+test_allocator
+test_blockstore
+test_shit
+osd_rmw_test

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "cpp-btree"]
+	path = cpp-btree
+	url = ../cpp-btree.git
+[submodule "json11"]
+	path = json11
+	url = ../json11.git

CLA-en.md

@@ -0,0 +1,115 @@
+## Contributor License Agreement
+
+> This Agreement is made in the Russian and English languages. **The English
+text of Agreement is for informational purposes only** and is not binding
+for the Parties.
+>
+> In the event of a conflict between the provisions of the Russian and
+English versions of this Agreement, the **Russian version shall prevail**.
+>
+> Russian version is published at https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-ru.md
+
+This document represents the offer of Filippov Vitaliy Vladimirovich
+("Author"), author and copyright holder of Vitastor software ("Program"),
+acknowledged by a certificate of Federal Service for Intellectual
+Property of Russian Federation (Rospatent) # 2021617829 dated 20 May 2021,
+to "Contributors" to conclude this license agreement as follows
+("Agreement" or "Offer").
+
+In accordance with Art. 435, Art. 438 of the Civil Code of the Russian
+Federation, this Agreement is an offer and in case of acceptance of the
+offer, an agreement is considered concluded on the conditions specified
+in the offer.
+
+1. Applicable Terms. \
+   1.1. "Official Repository" shall mean the computer storage, operated by
+        the Author, containing all prior and future versions of the Source
+        Code of the Program, at Internet addresses https://git.yourcmc.ru/vitalif/vitastor/
+        or https://github.com/vitalif/vitastor/. \
+   1.2. "Contributions" shall mean results of intellectual activity
+        (including, but not limited to, source code, libraries, components,
+        texts, documentation) which can be software or elements of the software
+        and which are provided by Contributors to the Author for inclusion
+        in the Program. \
+   1.3. "Contributor" shall mean a person who provides Contributions to
+        the Author and agrees with all provisions of this Agreement.
+        A Сontributor can be: 1) an individual; or 2) a legal entity or an
+        individual entrepreneur in case when an individual provides Contributions
+        on behalf of third parties, including on behalf of his employer.
+
+2. Subject of the Agreement. \
+   2.1. Subject of the Agreement shall be the Contributions sent to the Author by Contributors. \
+   2.2. The Contributor grants to the Author the right to use Contributions at his own
+        discretion and without any necessity to get a prior approval from Contributor or
+        any other third party in any way, under a simple (non-exclusive), royalty-free,
+        irrevocable license throughout the world by all means not contrary to law, in whole
+        or as a part of the Program, or other open-source or closed-source computer programs,
+        products or services (hereinafter -- the "License"), including, but not limited to: \
+        2.2.1. to execute Contributions and use them for any tasks; \
+        2.2.2. to publish and distribute Contributions in modified or unmodified form and/or to rent them; \
+        2.2.3. to modify Contributions, add comments, illustrations or any explanations to Contributions while using them; \
+        2.2.4. to create other results of intellectual activity based on Contributions, including derivative works and composite works; \
+        2.2.5. to translate Contributions into other languages, including other programming languages; \
+        2.2.6. to carry out rental and public display of Contributions; \
+        2.2.7. to use Contributions under the trade name and/or any trademark or any other label, or without it, as the Author thinks fit; \
+   2.3. The Contributor grants to the Author the right to sublicense any of the aforementioned
+        rights to third parties on any terms at the Author's discretion. \
+   2.4. The License is provided for the entire duration of Contributor's
+        exclusive intellectual property rights to the Contributions. \
+   2.5. The Contributor grants to the Author the right to decide how and where to mention,
+        or to not mention at all, the fact of his authorship, name, nickname and/or company
+        details when including Contributions into the Program or in any other computer
+        programs, products or services.
+
+3. Acceptance of the Offer \
+   3.1. The Contributor may provide Contributions to the Author in the form of
+        a "Pull Request" in an Official Repository of the Program or by any
+        other electronic means of communication, including, but not limited to,
+        E-mail or messenger applications. \
+   3.2. The acceptance of the Offer shall be the fact of provision of Contributions
+        to the Author by the Contributor by any means with the following remark:
+        “I accept Vitastor CLA agreement: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md”
+        or “Я принимаю соглашение Vitastor CLA: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-ru.md”. \
+   3.3. Date of acceptance of the Offer shall be the date of such provision.
+
+4. Rights and obligations of the parties. \
+   4.1. The Contributor reserves the right to use Contributions by any lawful means
+        not contrary to this Agreement. \
+   4.2. The Author has the right to refuse to include Contributions into the Program
+        at any moment with no explanation to the Contributor.
+
+5. Representations and Warranties. \
+   5.1. The person providing Contributions for the purpose of their inclusion
+        in the Program represents and warrants that he is the Contributor
+        or legally acts on the Contributor's behalf. Name or company details
+        of the Contributor shall be provided with the Contribution at the moment
+        of their provision to the Author. \
+   5.2. The Contributor represents and warrants that he legally owns exclusive
+        intellectual property rights to the Contributions. \
+   5.3. The Contributor represents and warrants that any further use of
+        Contributions by the Author as provided by Contributor under the terms
+        of the Agreement does not infringe on intellectual and other rights and
+        legitimate interests of third parties. \
+   5.4. The Contributor represents and warrants that he has all rights and legal
+        capacity needed to accept this Offer; \
+   5.5. The Contributor represents and warrants that Contributions don't
+        contain malware or any information considered illegal under the law
+        of Russian Federation.
+
+6. Termination of the Agreement \
+   6.1. The Agreement may be terminated at will of both Author and Contributor,
+        formalised in the written form or if the Agreement is terminated on
+        reasons prescribed by the law of Russian Federation.
+
+7. Final Clauses \
+   7.1. The Contributor may optionally sign the Agreement in the written form. \
+   7.2. The Agreement is deemed to become effective from the Date of signing of
+        the Agreement and until the expiration of Contributor's exclusive
+        intellectual property rights to the Contributions. \
+   7.3. The Author may unilaterally alter the Agreement without informing Contributors.
+        The new version of the document shall come into effect 3 (three) days after
+        being published in the Official Repository of the Program at Internet address
+        [https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md).
+        Contributors should keep informed about the actual version of the Agreement themselves. \
+   7.4. If the Author and the Contributor fail to agree on disputable issues,
+        disputes shall be referred to the Moscow Arbitration court.

CLA-ru.md

@@ -0,0 +1,108 @@
+## Лицензионное соглашение с участником
+
+> Данная Оферта написана в Русской и Английской версиях. **Версия на английском
+языке предоставляется в информационных целях** и не связывает стороны договора.
+>
+> В случае несоответствий между положениями Русской и Английской версий Договора,
+**Русская версия имеет приоритет**.
+>
+> Английская версия опубликована по адресу https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md
+
+Настоящий договор-оферта (далее по тексту – Оферта, Договор) адресована физическим
+и юридическим лицам (далее – Участникам) и является официальным публичным предложением
+Филиппова Виталия Владимировича (далее – Автора) программного обеспечения Vitastor,
+свидетельство Федеральной службы по интеллектуальной собственности (Роспатент) № 2021617829
+от 20 мая 2021 г. (далее – Программа) о нижеследующем:
+
+1. Термины и определения \
+   1.1. Репозиторий – электронное хранилище, содержащее исходный код Программы. \
+   1.2. Доработка – результат интеллектуальной деятельности Участника, включающий
+        в себя изменения или дополнения к исходному коду Программы, которые Участник
+        желает включить в состав Программы для дальнейшего использования и распространения
+        Автором и для этого направляет их Автору. \
+   1.3. Участник – физическое или юридическое лицо, вносящее Доработки в код Программы. \
+   1.4. ГК РФ – Гражданский кодекс Российской Федерации.
+
+2. Предмет оферты \
+   2.1. Предметом настоящей оферты являются Доработки, отправляемые Участником Автору. \
+   2.2. Участник предоставляет Автору право использовать Доработки по собственному усмотрению
+        и без необходимости предварительного согласования с Участником или иным третьим лицом
+        на условиях простой (неисключительной) безвозмездной безотзывной лицензии, полностью
+        или фрагментарно, в составе Программы или других программ, продуктов или сервисов
+        как с открытым, так и с закрытым исходным кодом, любыми способами, не противоречащими
+        закону, включая, но не ограничиваясь следующими: \
+        2.2.1. Запускать и использовать Доработки для выполнения любых задач; \
+        2.2.2. Распространять, импортировать и доводить Доработки до всеобщего сведения; \
+        2.2.3. Вносить в Доработки изменения, сокращения и дополнения, снабжать Доработки
+               при их использовании комментариями, иллюстрациями или пояснениями; \
+        2.2.4. Создавать на основе Доработок иные результаты интеллектуальной деятельности,
+               в том числе производные и составные произведения; \
+        2.2.5. Переводить Доработки на другие языки, в том числе на другие языки программирования; \
+        2.2.6. Осуществлять прокат и публичный показ Доработок; \
+        2.2.7. Использовать Доработки под любым фирменным наименованием, товарным знаком
+               (знаком обслуживания) или иным обозначением, или без такового. \
+   2.3. Участник предоставляет Автору право сублицензировать полученные права на Доработки
+        третьим лицам на любых условиях на усмотрение Автора. \
+   2.4. Участник предоставляет Автору права на Доработки на территории всего мира. \
+   2.5. Участник предоставляет Автору права на весь срок действия исключительного права
+        Участника на Доработки. \
+   2.6. Участник предоставляет Автору права на Доработки на безвозмездной основе. \
+   2.7. Участник разрешает Автору самостоятельно определять порядок, способ и
+        место указания его имени, реквизитов и/или псевдонима при включении
+        Доработок в состав Программы или других программ, продуктов или сервисов.
+
+3. Акцепт Оферты \
+   3.1. Участник может передавать Доработки в адрес Автора через зеркала официального
+        Репозитория Программы по адресам https://git.yourcmc.ru/vitalif/vitastor/ или
+        https://github.com/vitalif/vitastor/ в виде “запроса на слияние” (pull request),
+        либо в письменном виде или с помощью любых других электронных средств коммуникации,
+        например, электронной почты или мессенджеров. \
+   3.2. Факт передачи Участником Доработок в адрес Автора любым способом с одной из пометок
+        “I accept Vitastor CLA agreement: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-en.md”
+        или “Я принимаю соглашение Vitastor CLA: https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-ru.md”
+        является полным и безоговорочным акцептом (принятием) Участником условий настоящей
+        Оферты, т.е. Участник считается ознакомившимся с настоящим публичным договором и
+        в соответствии с ГК РФ признается лицом, вступившим с Автором в договорные отношения
+        на основании настоящей Оферты. \
+   3.3. Датой акцептирования настоящей Оферты считается дата такой передачи.
+
+4. Права и обязанности Сторон \
+   4.1. Участник сохраняет за собой право использовать Доработки любым законным
+        способом, не противоречащим настоящему Договору. \
+   4.2. Автор вправе отказать Участнику во включении Доработок в состав
+        Программы без объяснения причин в любой момент по своему усмотрению.
+
+5. Гарантии и заверения \
+   5.1. Лицо, направляющее Доработки для целей их включения в состав Программы,
+        гарантирует, что является Участником или представителем Участника. Имя или реквизиты
+        Участника должны быть указаны при их передаче в адрес Автора Программы. \
+   5.2. Участник гарантирует, что является законным обладателем исключительных прав
+        на Доработки. \
+   5.3. Участник гарантирует, что на момент акцептирования настоящей Оферты ему
+        ничего не известно (и не могло быть известно) о правах третьих лиц на
+        передаваемые Автору Доработки или их часть, которые могут быть нарушены
+        в связи с передачей Доработок по настоящему Договору. \
+   5.4. Участник гарантирует, что является дееспособным лицом и обладает всеми
+        необходимыми правами для заключения Договора. \
+   5.5. Участник гарантирует, что Доработки не содержат вредоносного ПО, а также
+        любой другой информации, запрещённой к распространению по законам Российской
+        Федерации.
+
+6. Прекращение действия оферты \
+   6.1. Действие настоящего договора может быть прекращено по соглашению сторон,
+        оформленному в письменном виде, а также вследствие его расторжения по основаниям,
+        предусмотренным законом.
+
+7. Заключительные положения \
+   7.1. Участник вправе по желанию подписать настоящий Договор в письменном виде. \
+   7.2. Настоящий договор действует с момента его заключения и до истечения срока
+        действия исключительных прав Участника на Доработки. \
+   7.3. Автор имеет право в одностороннем порядке вносить изменения и дополнения в договор
+        без специального уведомления об этом Участников. Новая редакция документа вступает
+        в силу через 3 (Три) календарных дня со дня опубликования в официальном Репозитории
+        Программы по адресу в сети Интернет
+        [https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-ru.md](https://git.yourcmc.ru/vitalif/vitastor/src/branch/master/CLA-ru.md).
+        Участники самостоятельно отслеживают действующие условия Оферты. \
+   7.4. Все споры, возникающие между сторонами в процессе их взаимодействия по настоящему
+        договору, решаются путём переговоров. В случае невозможности урегулирования споров
+        переговорным порядком стороны разрешают их в Арбитражном суде г.Москвы.

CMakeLists.txt

@@ -0,0 +1,7 @@
+cmake_minimum_required(VERSION 2.8.12)
+
+project(vitastor)
+
+set(VERSION "1.4.4")
+
+add_subdirectory(src)

GPL-2.0.txt

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
+
+All server-side code (OSD, Monitor and so on) is licensed under the terms of
+Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
+GNU GPLv3.0 with the additional "Network Interaction" clause which requires
+opensourcing all programs directly or indirectly interacting with Vitastor
+through a computer network and expressly designed to be used in conjunction
+with it ("Proxy Programs"). Proxy Programs may be made public not only under
+the terms of the same license, but also under the terms of any GPL-Compatible
+Free Software License, as listed by the Free Software Foundation.
+This is a stricter copyleft license than the Affero GPL.
+
+Please note that VNPL doesn't require you to open the code of proprietary
+software running inside a VM if it's not specially designed to be used with
+Vitastor.
+
+Basically, you can't use the software in a proprietary environment to provide
+its functionality to users without opensourcing all intermediary components
+standing between the user and Vitastor or purchasing a commercial license
+from the author 😀.
+
+Client libraries (cluster_client and so on) are dual-licensed under the same
+VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+software like QEMU and fio.
+
+You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
+GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).

Makefile

@@ -1,64 +0,0 @@
-BLOCKSTORE_OBJS := allocator.o blockstore.o blockstore_impl.o blockstore_init.o blockstore_open.o blockstore_journal.o blockstore_read.o \
-	blockstore_write.o blockstore_sync.o blockstore_stable.o blockstore_rollback.o blockstore_flush.o crc32c.o ringloop.o timerfd_interval.o
-# -fsanitize=address
-CXXFLAGS := -g -O3 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fPIC -fdiagnostics-color=always
-all: $(BLOCKSTORE_OBJS) libfio_blockstore.so osd libfio_sec_osd.so test_blockstore stub_osd osd_test
-clean:
-	rm -f *.o
-
-crc32c.o: crc32c.c
-	g++ $(CXXFLAGS) -c -o $@ $<
-json11.o: json11/json11.cpp
-	g++ $(CXXFLAGS) -c -o json11.o json11/json11.cpp
-allocator.o: allocator.cpp allocator.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-ringloop.o: ringloop.cpp ringloop.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-timerfd_interval.o: timerfd_interval.cpp timerfd_interval.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-
-%.o: %.cpp allocator.h blockstore_flush.h blockstore.h blockstore_impl.h blockstore_init.h blockstore_journal.h crc32c.h ringloop.h timerfd_interval.h object_id.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-
-libblockstore.so: $(BLOCKSTORE_OBJS)
-	g++ $(CXXFLAGS) -o libblockstore.so -shared $(BLOCKSTORE_OBJS) -ltcmalloc_minimal -luring
-libfio_blockstore.so: ./libblockstore.so fio_engine.cpp json11.o
-	g++ $(CXXFLAGS) -shared -o libfio_blockstore.so fio_engine.cpp json11.o ./libblockstore.so -ltcmalloc_minimal -luring
-
-OSD_OBJS := osd.o osd_secondary.o osd_receive.o osd_send.o osd_peering.o osd_peering_pg.o osd_primary.o osd_rmw.o json11.o timerfd_interval.o
-osd_secondary.o: osd_secondary.cpp osd.h osd_ops.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_receive.o: osd_receive.cpp osd.h osd_ops.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_send.o: osd_send.cpp osd.h osd_ops.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering.o: osd_peering.cpp osd.h osd_ops.h osd_peering_pg.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_peering_pg.o: osd_peering_pg.cpp object_id.h osd_peering_pg.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw.o: osd_rmw.cpp osd_rmw.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_rmw_test: osd_rmw_test.cpp osd_rmw.cpp osd_rmw.h xor.h
-	g++ $(CXXFLAGS) -o $@ $<
-osd_primary.o: osd_primary.cpp osd.h osd_ops.h osd_peering_pg.h xor.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd.o: osd.cpp osd.h osd_ops.h osd_peering_pg.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd: ./libblockstore.so osd_main.cpp osd.h osd_ops.h $(OSD_OBJS)
-	g++ $(CXXFLAGS) -o osd osd_main.cpp $(OSD_OBJS) ./libblockstore.so -ltcmalloc_minimal -luring
-stub_osd: stub_osd.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o stub_osd stub_osd.cpp rw_blocking.o -ltcmalloc_minimal
-rw_blocking.o: rw_blocking.cpp rw_blocking.h
-	g++ $(CXXFLAGS) -c -o $@ $<
-osd_test: osd_test.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -o osd_test osd_test.cpp rw_blocking.o -ltcmalloc_minimal
-
-libfio_sec_osd.so: fio_sec_osd.cpp osd_ops.h rw_blocking.o
-	g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o libfio_sec_osd.so fio_sec_osd.cpp rw_blocking.o -luring
-
-test_blockstore: ./libblockstore.so test_blockstore.cpp
-	g++ $(CXXFLAGS) -o test_blockstore test_blockstore.cpp ./libblockstore.so -ltcmalloc_minimal -luring
-test: test.cpp osd_peering_pg.o
-	g++ $(CXXFLAGS) -o test test.cpp osd_peering_pg.o -luring
-test_allocator: test_allocator.cpp allocator.o
-	g++ $(CXXFLAGS) -o test_allocator test_allocator.cpp allocator.o

README-ru.md

@@ -0,0 +1,102 @@
+## Vitastor
+
+[Read English version](README.md)
+
+## Идея
+
+Вернём былую скорость кластерному блочному хранилищу!
+
+Vitastor - распределённая блочная SDS (программная СХД), прямой аналог Ceph RBD и
+внутренних СХД популярных облачных провайдеров. Однако, в отличие от них, Vitastor
+быстрый и при этом простой. Только пока маленький :-).
+
+Vitastor архитектурно похож на Ceph, что означает атомарность и строгую консистентность,
+репликацию через первичный OSD, симметричную кластеризацию без единой точки отказа
+и автоматическое распределение данных по любому числу дисков любого размера с настраиваемыми схемами
+избыточности - репликацией или с произвольными кодами коррекции ошибок.
+
+Vitastor нацелен в первую очередь на SSD и SSD+HDD кластеры с как минимум 10 Гбит/с сетью, поддерживает
+TCP и RDMA и на хорошем железе может достигать задержки 4 КБ чтения и записи на уровне ~0.1 мс,
+что примерно в 10 раз быстрее, чем Ceph и другие популярные программные СХД.
+
+Vitastor поддерживает QEMU-драйвер, протоколы NBD и NFS, драйверы OpenStack, Proxmox, Kubernetes.
+Другие драйверы могут также быть легко реализованы.
+
+Подробности смотрите в документации по ссылкам ниже.
+
+## Презентации и записи докладов
+
+- DevOpsConf'2021: презентация ([на русском](https://vitastor.io/presentation/devopsconf/devopsconf.html),
+  [на английском](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
+  [видео](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: презентация ([на русском](https://vitastor.io/presentation/highload/highload.html)),
+  [видео](https://vitastor.io/presentation/highload/talk.webm)
+
+## Документация
+
+- Введение
+  - [Быстрый старт](docs/intro/quickstart.ru.md)
+  - [Возможности](docs/intro/features.ru.md)
+  - [Архитектура](docs/intro/architecture.ru.md)
+  - [Автор и лицензия](docs/intro/author.ru.md)
+- Установка
+  - [Пакеты](docs/installation/packages.ru.md)
+  - [Proxmox](docs/installation/proxmox.ru.md)
+  - [OpenStack](docs/installation/openstack.ru.md)
+  - [Kubernetes CSI](docs/installation/kubernetes.ru.md)
+  - [Сборка из исходных кодов](docs/installation/source.ru.md)
+- Конфигурация
+  - [Обзор](docs/config.ru.md)
+  - Параметры
+    - [Общие](docs/config/common.ru.md)
+    - [Сетевые](docs/config/network.ru.md)
+    - [Клиентский код](docs/config/client.en.md)
+    - [Глобальные дисковые параметры](docs/config/layout-cluster.ru.md)
+    - [Дисковые параметры OSD](docs/config/layout-osd.ru.md)
+    - [Прочие параметры OSD](docs/config/osd.ru.md)
+    - [Параметры мониторов](docs/config/monitor.ru.md)
+  - [Настройки пулов](docs/config/pool.ru.md)
+  - [Метаданные образов в etcd](docs/config/inode.ru.md)
+- Использование
+  - [vitastor-cli](docs/usage/cli.ru.md) (консольный интерфейс)
+  - [vitastor-disk](docs/usage/disk.ru.md) (управление дисками)
+  - [fio](docs/usage/fio.ru.md) для тестов производительности
+  - [NBD](docs/usage/nbd.ru.md) для монтирования ядром
+  - [QEMU и qemu-img](docs/usage/qemu.ru.md)
+  - [NFS](docs/usage/nfs.ru.md)-прокси для VMWare и подобных
+- Производительность
+  - [Понимание сути производительности](docs/performance/understanding.ru.md)
+  - [Теоретический максимум](docs/performance/theoretical.ru.md)
+  - [Пример сравнения с Ceph](docs/performance/comparison1.ru.md)
+
+## Автор и лицензия
+
+Автор: Виталий Филиппов (vitalif [at] yourcmc.ru), 2019+
+
+Заходите в Telegram-чат Vitastor: https://t.me/vitastor
+
+Лицензия: VNPL 1.1 на серверный код и двойная VNPL 1.1 + GPL 2.0+ на клиентский.
+
+VNPL - "сетевой копилефт", собственная свободная копилефт-лицензия
+Vitastor Network Public License 1.1, основанная на GNU GPL 3.0 с дополнительным
+условием "Сетевого взаимодействия", требующим распространять все программы,
+специально разработанные для использования вместе с Vitastor и взаимодействующие
+с ним по сети, под лицензией VNPL или под любой другой свободной лицензией.
+
+Идея VNPL - расширение действия копилефта не только на модули, явным образом
+связываемые с кодом Vitastor, но также на модули, оформленные в виде микросервисов
+и взаимодействующие с ним по сети.
+
+Таким образом, если вы хотите построить на основе Vitastor сервис, содержаший
+компоненты с закрытым кодом, взаимодействующие с Vitastor, вам нужна коммерческая
+лицензия от автора 😀.
+
+На Windows и любое другое ПО, не разработанное *специально* для использования
+вместе с Vitastor, никакие ограничения не накладываются.
+
+Клиентские библиотеки распространяются на условиях двойной лицензии VNPL 1.0
+и также на условиях GNU GPL 2.0 или более поздней версии. Так сделано в целях
+совместимости с таким ПО, как QEMU и fio.
+
+Вы можете найти полный текст VNPL 1.1 на английском языке в файле [VNPL-1.1.txt](VNPL-1.1.txt),
+VNPL 1.1 на русском языке в файле [VNPL-1.1-RU.txt](VNPL-1.1-RU.txt), а GPL 2.0 в файле [GPL-2.0.txt](GPL-2.0.txt).

README.md

@@ -0,0 +1,102 @@
+# Vitastor
+
+[Читать на русском](README-ru.md)
+
+## The Idea
+
+Make Clustered Block Storage Fast Again.
+
+Vitastor is a distributed block SDS, direct replacement of Ceph RBD and internal SDS's
+of public clouds. However, in contrast to them, Vitastor is fast and simple at the same time.
+The only thing is it's slightly young :-).
+
+Vitastor is architecturally similar to Ceph which means strong consistency,
+primary-replication, symmetric clustering and automatic data distribution over any
+number of drives of any size with configurable redundancy (replication or erasure codes/XOR).
+
+Vitastor targets primarily SSD and SSD+HDD clusters with at least 10 Gbit/s network,
+supports TCP and RDMA and may achieve 4 KB read and write latency as low as ~0.1 ms
+with proper hardware which is ~10 times faster than other popular SDS's like Ceph
+or internal systems of public clouds.
+
+Vitastor supports QEMU, NBD, NFS protocols, OpenStack, Proxmox, Kubernetes drivers.
+More drivers may be created easily.
+
+Read more details below in the documentation.
+
+## Talks and presentations
+
+- DevOpsConf'2021: presentation ([in Russian](https://vitastor.io/presentation/devopsconf/devopsconf.html),
+  [in English](https://vitastor.io/presentation/devopsconf/devopsconf_en.html)),
+  [video](https://vitastor.io/presentation/devopsconf/talk.webm)
+- Highload'2022: presentation ([in Russian](https://vitastor.io/presentation/highload/highload.html)),
+  [video](https://vitastor.io/presentation/highload/talk.webm)
+
+## Documentation
+
+- Introduction
+  - [Quick Start](docs/intro/quickstart.en.md)
+  - [Features](docs/intro/features.en.md)
+  - [Architecture](docs/intro/architecture.en.md)
+  - [Author and license](docs/intro/author.en.md)
+- Installation
+  - [Packages](docs/installation/packages.en.md)
+  - [Proxmox](docs/installation/proxmox.en.md)
+  - [OpenStack](docs/installation/openstack.en.md)
+  - [Kubernetes CSI](docs/installation/kubernetes.en.md)
+  - [Building from Source](docs/installation/source.en.md)
+- Configuration
+  - [Overview](docs/config.en.md)
+  - Parameter Reference
+    - [Common](docs/config/common.en.md)
+    - [Network](docs/config/network.en.md)
+    - [Client](docs/config/client.en.md)
+    - [Global Disk Layout](docs/config/layout-cluster.en.md)
+    - [OSD Disk Layout](docs/config/layout-osd.en.md)
+    - [OSD Runtime Parameters](docs/config/osd.en.md)
+    - [Monitor](docs/config/monitor.en.md)
+  - [Pool configuration](docs/config/pool.en.md)
+  - [Image metadata in etcd](docs/config/inode.en.md)
+- Usage
+  - [vitastor-cli](docs/usage/cli.en.md) (command-line interface)
+  - [vitastor-disk](docs/usage/disk.en.md) (disk management tool)
+  - [fio](docs/usage/fio.en.md) for benchmarks
+  - [NBD](docs/usage/nbd.en.md) for kernel mounts
+  - [QEMU and qemu-img](docs/usage/qemu.en.md)
+  - [NFS](docs/usage/nfs.en.md) emulator for VMWare and similar
+- Performance
+  - [Understanding storage performance](docs/performance/understanding.en.md)
+  - [Theoretical performance](docs/performance/theoretical.en.md)
+  - [Example comparison with Ceph](docs/performance/comparison1.en.md)
+
+## Author and License
+
+Copyright (c) Vitaliy Filippov (vitalif [at] yourcmc.ru), 2019+
+
+Join Vitastor Telegram Chat: https://t.me/vitastor
+
+All server-side code (OSD, Monitor and so on) is licensed under the terms of
+Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
+GNU GPLv3.0 with the additional "Network Interaction" clause which requires
+opensourcing all programs directly or indirectly interacting with Vitastor
+through a computer network and expressly designed to be used in conjunction
+with it ("Proxy Programs"). Proxy Programs may be made public not only under
+the terms of the same license, but also under the terms of any GPL-Compatible
+Free Software License, as listed by the Free Software Foundation.
+This is a stricter copyleft license than the Affero GPL.
+
+Please note that VNPL doesn't require you to open the code of proprietary
+software running inside a VM if it's not specially designed to be used with
+Vitastor.
+
+Basically, you can't use the software in a proprietary environment to provide
+its functionality to users without opensourcing all intermediary components
+standing between the user and Vitastor or purchasing a commercial license
+from the author 😀.
+
+Client libraries (cluster_client and so on) are dual-licensed under the same
+VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+software like QEMU and fio.
+
+You can find the full text of VNPL-1.1 in the file [VNPL-1.1.txt](VNPL-1.1.txt).
+GPL 2.0 is also included in this repository as [GPL-2.0.txt](GPL-2.0.txt).

VNPL-1.1-RU.txt

@@ -0,0 +1,680 @@
+                СЕТЕВАЯ ПУБЛИЧНАЯ ЛИЦЕНЗИЯ VITASTOR
+                  VITASTOR NETWORK PUBLIC LICENSE
+                   Версия 1.1, от 6 февраля 2021
+
+ Автор лицензии: Виталий Филиппов <vitalif@yourcmc.ru>, 2021 год
+ Каждый имеет право копировать и распространять точные копии этой
+ лицензии, но без внесения изменений.
+
+                            ПРЕАМБУЛА
+
+  Сетевая Публичная Лицензия Vitastor - это свободная "копилефт" лицензия для
+для программного обеспечения (ПО) и других видов произведений, специально
+разработанная, чтобы гарантировать кооперацию с сообществом при разработке
+сетевых приложений.
+
+  Большинство лицензий на программное обеспечение и другие произведения
+спроектированы так, чтобы лишить Вас свободы делиться ими и изменять их.
+Сетевая Публичная Лицензия Vitastor, напротив, разработана с целью
+гарантировать Ваше право распространять и вносить изменения во все версии
+программного обеспечения -- для уверенности, что ПО останется свободным для
+всех пользователей.
+
+  Когда мы говорим о свободном ПО, мы имеем в виду свободу использования, а не
+бесплатность. Свободные лицензии, такие, как Сетевая Публичная Лицензия
+Vitastor, составлены для того, чтобы убедиться, что у Вас есть право
+распространять копии свободного ПО (и взимать плату за них, если Вы хотите),
+что Вы получаете исходные тексты или можете получить их, если захотите, что Вы
+можете изменять программное обеспечение или использовать его части в новых
+свободных программах, и что Вы знаете о своем праве делать всё это.
+
+  Разработчики, использующие Сетевую Публичную Лицензию Vitastor, гарантируют
+Ваши права при помощи следующих мер: (1) закрепляют авторское право на
+программное обеспечение, и (2) предлагают Вам принять условия настоящей
+Лицензии, закрепляющей Ваше право на создание копий, распространение и (или)
+модификацию программного обеспечения.
+
+  Еще одно преимущество защиты свободы всех пользователей заключается в том,
+что улучшения, сделанные в разных версиях программы, при их широком
+распространении становятся доступными для использования другими разработчиками.
+Многие разработчики программного обеспечения воодушевляются этим
+сотрудничеством и пользуются его преимуществами. Однако, если программное
+обеспечение используется на сетевых серверах, данный результат не всегда
+достигается. Генеральная публичная лицензия GNU разрешает создание измененных
+версий и предоставление неограниченного доступа к ним, не делая общедоступным
+их исходный текст. Даже генеральная публичная лицензия GNU Affero разрешает
+использование модифицированной версии свободной программы в закрытой среде, где
+внешние пользователи взаимодействуют с ней только через закрытый промежуточный
+интерфейс (прокси), опять же, без открытия в свободный публичный доступ как
+самой программы, так и прокси.
+
+  Сетевая Публичная Лицензия Vitastor разработана специально, чтобы
+гарантировать, что в таких случаях и модифицированная версия программы, и
+прокси останутся доступными сообществу. Для этого лицензия требует от
+операторов сетевых серверов предоставлять исходный код оригинальной программы,
+а также всех других программ, взаимодействующих с ней на их серверах,
+пользователям этих серверов, на условиях свободных лицензий. Таким образом,
+публичное использование изменённой версии ПО на сервере, прямо или косвенно
+доступном пользователям, даёт пользователям доступ к исходным кодам изменённой
+версии.
+
+  Детальные определения используемых терминов и описание условий копирования,
+распространения и внесения изменений приведены ниже.
+
+                        ТЕРМИНЫ И УСЛОВИЯ
+
+  0. Определения.
+
+  "Настоящая Лицензия" -- версия 1.1 Сетевой Публичной Лицензии Vitastor.
+
+  Под "Авторским правом" понимаются все законы, сходные с авторско-правовыми,
+которые применяются к любым видам работ, например, к топологиям микросхем.
+
+  Термином "Программа" обозначается любое охраноспособное произведение,
+используемое в соответствии с настоящей Лицензией. Лицензиат именуется "Вы".
+"Лицензиаты" и "получатели" могут быть как физическими лицами, так и
+организациями.
+
+  "Внесение изменений" в произведение означает копирование или адаптацию
+произведения целиком или в части, способом, требующим разрешения
+правообладателя, за исключением изготовления его точной копии. Получившееся
+произведение называется "измененной версией" предыдущего произведения или
+произведением, "основанным на" более ранней работе.
+
+  Термином "Лицензионное произведение" обозначается неизмененная Программа или
+произведение, основанное на Программе.
+
+  "Распространение" произведения означает совершение с ним действий, которые
+при отсутствии разрешения сделают Вас прямо или косвенно ответственным за
+нарушение действующего закона об авторском праве, за исключением запуска на
+компьютере или изменения копии, созданной в личных целях. Распространение
+включает в себя копирование, раздачу копий (с изменениями или без них),
+доведение до всеобщего сведения, а в некоторых странах -- и другие действия.
+
+  "Передача" произведения означает любой вид распространения, который позволяет
+другим лицам создавать или получать копии произведения. Обычное взаимодействие
+с пользователем через компьютерную сеть без создания копии передачей не
+является.
+
+  Интерактивный интерфейс пользователя должен отображать "Информация об
+авторском праве", достаточную для того, чтобы (1) обеспечить отображение
+соответствующего уведомления об авторских правах и (2) сообщить пользователю
+о том, что ему не предоставляются никакие гарантии на произведение (за
+исключением явным образом предоставленных гарантий), о том, что лицензиаты
+могут передавать произведение на условиях, описанных в настоящей Лицензии,
+а также о том, как ознакомиться с текстом настоящей Лицензии. Если интерфейс
+предоставляет собой список пользовательских команд или настроек, наподобие
+меню, это требование считается выполненным при наличии явно выделенного
+пункта в таком меню.
+
+  1. Исходный текст.
+
+  Под "Исходным текстом" понимается произведение в форме, которая более всего
+подходит для внесения в него изменений. "Объектным кодом" называется
+произведение в любой иной форме.
+
+  "Стандартный интерфейс" -- интерфейс, который либо является общепринятым
+стандартом, введенным общепризнанным органом по стандартизации, либо, в случае
+интерфейсов, характерных для конкретного языка программирования -- тот,
+который широко используется разработчиками, пишущими программы на этом языке.
+
+  "Системные библиотеки" исполняемого произведения включают в себя то, что не
+относится к произведению в целом и при этом (a) входит в обычный комплект
+Основного компонента, но при этом не является его частью и (b) служит только
+для обеспечения работы с этим Основным компонентом или для реализации
+Стандартного интерфейса, для которого существует общедоступная реализация,
+опубликованная в виде исходного текста. "Основным компонентом" в данном
+контексте назван главный существенный компонент (ядро, оконная система и т.д.)
+определенной операционной системы (если она используется), под управлением
+которой функционирует исполняемое произведение, либо компилятор, используемый
+для создания произведения или интерпретатор объектного кода, используемый для
+его запуска.
+
+  "Полный исходный текст" для произведения в форме объектного кода -- весь
+исходный текст, необходимый для создания, установки и (для исполняемого
+произведения) функционирования объектного кода, а также модификации
+произведения, включая сценарии, контролирующие эти действия. Однако он не
+включает в себя Системные библиотеки, необходимые для функционирования
+произведения, инструменты общего назначения или общедоступные свободные
+программы, которые используются в неизменном виде для выполнения этих
+действий, но не являются частью произведения. Полный исходный текст включает
+в себя, например, файлы описания интерфейса, прилагаемые к файлам исходного
+текста произведения, а также исходные тексты общих библиотек и динамически
+связанных подпрограмм, которые требуются для функционирования произведения
+и разработаны специально для этого, например, для прямой передачи данных
+или управления потоками между этими подпрограммами и другими частями
+произведения. Полный исходный текст не включает в себя то, что пользователи
+могут сгенерировать автоматически из других частей Полного исходного текста.
+Полным исходным текстом для произведения в форме исходных текстов является
+само это произведение.
+
+  2. Основные права.
+
+  Все права, предоставленные на основании настоящей Лицензии, действуют в
+течение срока действия авторских прав на Программу и не могут быть отозваны
+при условии, что сформулированные в ней условия соблюдены. Настоящая Лицензия
+однозначно подтверждает Ваши неограниченные права на запуск неизмененной
+Программы. Настоящая Лицензия распространяется на результаты функционирования
+Лицензионного произведения только в том случае, если они, учитывая их
+содержание, сами являются частью Лицензионного произведения. Настоящая
+Лицензия подтверждает Ваши права на  свободное использование произведения
+или другие аналогичные полномочия, предусмотренные действующим
+законодательством об авторском праве.
+
+  Если Вы не осуществляете обычную передачу Лицензионного произведения, то
+можете как угодно создавать, запускать и распространять его копии до тех пор,
+пока ваша Лицензия сохраняет силу. Вы можете передавать Лицензионные
+произведения третьим лицам исключительно для того, чтобы они внесли в них
+изменения для Вас или предоставили Вам возможность их запуска, при условии,
+что Вы соглашаетесь с условиями настоящей Лицензии при передаче всех
+материалов, авторскими правами на которые Вы не обладаете. Лица, создающие
+или запускающие Лицензионные произведения для Вас, должны делать это
+исключительно от Вашего имени, под Вашим руководством и контролем, на
+условиях, которые запрещают им создание без Вашей санкции каких-либо копий
+материалов, на которые Вы обладаете авторским правом.
+
+  Любая другая передача разрешается исключительно при соблюдении описанных
+ниже условий. Сублицензирование не допускается; раздел 10 делает его не нужным.
+
+  3. Защита прав пользователей от законов, запрещающих обход технических средств.
+
+  Ни одно Лицензионное произведение не должно считаться содержащим эффективные
+технические средства, удовлетворяющие требованиям любого действующего закона,
+принятого для исполнения обязательств, предусмотренных статьей 11 Договора ВОИС
+по авторскому праву от 20 декабря 1996 года или аналогичных законов,
+запрещающих или ограничивающих обход таких технических средств.
+
+  При передаче Лицензионного произведения Вы отказываетесь от всех
+предоставляемых законом полномочий по запрету обхода технических средств,
+используемых авторами в связи с осуществлением их прав, признавая, что такой
+обход находится в рамках осуществления прав на использование Лицензионного
+произведения, предоставленных настоящей Лицензией; также Вы отказываетесь
+от любых попыток ограничить функционирование произведения или внесение в него
+изменений, направленных на реализацию предоставленных Вам законом прав на
+запрет пользователю обхода технических средств.
+
+  4. Передача неизмененных копий.
+
+  Вы можете передавать точные копии исходных текстов Программы в том виде,
+в котором Вы их получили, на любом носителе, при условии, что Вы прилагаете
+к каждой копии соответствующее уведомление об авторских правах способом,
+обеспечивающим ознакомление с ним пользователя; сохраняете все уведомления
+о том, что к тексту применима настоящая Лицензия и любые ограничения,
+добавленные в соответствии с разделом 7; сохраняете все уведомления об
+отсутствии каких-либо гарантий; предоставляете всем получателям вместе с
+Программой копию настоящей Лицензии.
+
+  Вы можете установить любую цену за каждую копию, которую Вы передаете,
+или распространять копии бесплатно; также Вы можете предложить поддержку
+или гарантию за отдельную плату.
+
+  5. Передача измененных исходных текстов.
+
+  Вы можете передавать исходный текст произведения, основанного на Программе,
+или изменений, необходимых для того, чтобы получить его из Программы, на
+условиях, описанных в разделе 4, при соблюдении следующих условий:
+
+    а) Произведение должно содержать уведомления о произведенных Вами
+    изменениях с указанием их даты, сделанные способом, обеспечивающим
+    ознакомление с ними пользователя.
+
+    b) Произведение должно содержать уведомление о том, что оно
+    распространяется на условиях настоящей Лицензии, а также об условиях,
+    добавленных в соответствии с разделом 7, сделанное способом,
+    обеспечивающим ознакомление с ним пользователя. Данное требование имеет
+    приоритет над требованиями раздела 4 "оставлять нетронутыми все
+    уведомления".
+
+    c) Вы должны передать на условиях настоящей Лицензии всю работу целиком
+    любому лицу, которое приобретает копию. Таким образом, настоящая Лицензия
+    вместе с любыми применимыми условиями раздела 7 будет применяться к
+    произведению в целом и всем его частям, независимо от их комплектности.
+    Настоящая Лицензия не дает права на лицензирование произведения на любых
+    других условиях, но это не лишает законной силы такое разрешение, если Вы
+    получили его отдельно.
+
+    d) Если произведение имеет интерактивные пользовательские интерфейсы,
+    каждый из них должен отображать Информацию об авторском праве; однако,
+    если Программа имеет пользовательские интерфейсы, которые не отображают
+    информацию об авторском праве, от Вашего произведения этого также не
+    требуется.
+
+  Включение Лицензионного произведения в подборку на разделе хранения данных
+или на носителе, на котором распространяется произведение, вместе с другими
+отдельными самостоятельными произведениями, которые по своей природе не
+являются переработкой Лицензионного произведения и не объединены с ним,
+например, в программный комплекс, называется "набором", если авторские права
+на подборку не используются для ограничения доступа к ней или законных прав
+её пользователей сверх того, что предусматривают лицензии на отдельные
+произведения. Включение Лицензионного произведения в набор не влечет применения
+положений настоящей Лицензии к остальным его частям.
+
+  6. Передача произведения в формах, не относящихся к исходному тексту.
+
+  Вы можете передавать Лицензионное произведение в виде объектного кода в
+соответствии с положениями разделов 4 и 5, при условии, что Вы также передаете
+машиночитаемый Полный исходный текст в соответствии с условиями настоящей
+Лицензии, одним из следующих способов:
+
+    а) Передавая объектный код или содержащий его материальный продукт (включая
+    распределенный материальный носитель), с приложением Полного исходного
+    текста наматериальном носителе, обычно используемом для обмена программным
+    обеспечением.
+
+    b) Передавая объектный код или содержащий его материальный продукт (включая
+    носитель, на котором распространяется произведение), с письменным
+    предложением, действительным в течение не менее трех лет либо до тех пор,
+    пока Вы предоставляете запасные части или поддержку для данного продукта,
+    о передаче любому обладателю объектного кода (1) копии Полного исходного
+    текста для всего программного обеспечения, содержащегося в продукте, на
+    которое распространяется действие настоящей Лицензии, на физическом
+    носителе, обычно используемом для обмена программным обеспечением, по цене,
+    не превышающей разумных затрат на передачу копии, или (2) доступа к Полному
+    исходному тексту с возможностью его копирования с сетевого сервера без
+    взимания платы.
+
+    с) Передавая отдельные копии объектного кода с письменной копией предложения
+    о предоставлении Полного исходного текста. Этот вариант допускается только
+    в отдельных случаях при распространении без извлечения прибыли, и только
+    если Вы получили объектный код с таким предложением в соответствии
+    с пунктом 6b.
+
+    d) Передавая объектный код посредством предоставления доступа к нему по
+    определенному адресу (бесплатно или за дополнительную плату), и предлагая
+    эквивалентный доступ к Полному исходному тексту таким же способом по тому же
+    адресу без какой-либо дополнительной оплаты. От Вас не требуется принуждать
+    получателей копировать Полный исходный текст вместе с объектным кодом. Если
+    объектный код размещен на сетевом сервере, Полный исходный текст может
+    находиться на другом сервере (управляемом Вами или третьим лицом), который
+    предоставляет аналогичную возможность копирования; при этом Вы должны четко
+    указать рядом с объектным кодом способ получения Полного исходного текста.
+    Независимо от того, на каком сервере расположен Полный исходный текст, Вы
+    обязаны убедиться в том, что он будет распространяться в течение времени,
+    необходимого для соблюдения этих требований.
+
+    e) Передавая объектный код с использованием одноранговой (пиринговой) сети,
+    при условии информирования других пользователей сети о том, где можно
+    бесплатно получить объектный код и Полный исходный текст произведения
+    способом, описанным в пункте 6d.
+
+  Не нужно включать в передаваемый объектный код его отделимые части, исходные
+тексты которых не входят в состав Полного исходного текста, такие как Системные
+библиотеки.
+
+  "Потребительский товар" это либо (1) "товар, предназначенный для личных нужд",
+под которым понимается любое материальное личное имущество, которое обычно
+используется для личных, семейных или домашних целей, или (2) что-либо
+спроектированное или продающееся для использования в жилище. При определении
+того, предназначен ли товар для личных нужд, сомнения должны толковаться в
+пользу положительного ответа на этот вопрос. Применительно к конкретному
+товару, используемому конкретным пользователем, под выражением "обычно
+используется" имеется в виду способ, которым данный вид товаров преимущественно
+или как правило используется, независимо от статуса конкретного пользователя
+или способа, которым конкретный пользователь использует, предполагает или
+будет использовать товар. Товар относится к предназначенным для личных нужд
+независимо от того, насколько часто он используется в коммерческой
+деятельности, промышленности или иной сфере, не относящейся к личным нуждам,
+за исключением случая, когда использование в этой сфере представляет собой
+единственный основной способ использования такого товара.
+
+  "Информация, необходимая для установки" Потребительского товара -- любые
+методы, процедуры, сведения, необходимые для авторизации, или другая
+информация, необходимая для установки и запуска в Потребительском товаре
+измененных версий Лицензионного произведения, полученных при изменении
+Полного исходного текста. Данная информация должна быть достаточной для
+того, чтобы обеспечить возможность внесения в исходный текст изменений,
+не приводящих к ограничению или нарушению его дальнейшей работоспособности.
+
+  Если вместе с Потребительским товаром или специально для использования
+в нём Вы передаете произведение в виде объектного кода на условиях, описанных
+в данном разделе, и такая передача является частью сделки, по которой право
+владения и пользования Потребительским товаром переходит к получателю
+пожизненно или на определенный срок (независимо от признаков сделки), Полный
+исходный текст, передаваемый согласно данному разделу, должен сопровождаться
+Информацией, необходимой для установки. Но это требование не применяется,
+если ни Вы, ни какое-либо третье лицо не сохраняет за собой возможности
+установки измененного объектного кода на Потребительский товар (например,
+произведение было установлено в постоянную память).
+
+  Требование о предоставлении Информации, необходимой для установки, не
+включает в себя требование продолжения оказания услуг по поддержке,
+предоставления гарантии или обновлений для произведения, которое было изменено
+или установлено получателем, либо для Потребительского товара, в котором оно
+было изменено или на который оно было установлено. В доступе к сети может быть
+отказано, если само внесение изменений существенно и негативно влияет на
+работу сети, нарушает правила обмена данными или не поддерживает протоколы для
+обмена данными по сети.
+
+  Передаваемый в соответствии с данным разделом Полный исходный текст и
+предоставленная Информация, необходимая для установки, должны быть записаны в
+формате, который имеет общедоступное описание (и общедоступную реализацию,
+опубликованную в форме исходного текста) и не должны требовать никаких
+специальных паролей или ключей для распаковки, чтения или копирования.
+
+  7. Дополнительные условия.
+
+  "Дополнительными разрешениями" называются условия, которые дополняют условия
+настоящей Лицензии, делая исключения из одного или нескольких её положений.
+Дополнительные разрешения, которые применимы ко всей Программе, должны
+рассматриваться как часть настоящей Лицензии, в той степени, в которой они
+соответствуют действующему законодательству. Если дополнительные разрешения
+применяются только к части Программы, эта часть может быть использована отдельно
+на измененных условиях, но вся Программа продолжает использоваться на условиях
+настоящей Лицензии без учета дополнительных разрешений.
+
+  Когда Вы передаете копию Лицензионного произведения, Вы можете по своему
+усмотрению исключить любые дополнительные разрешения, примененные к этой копии
+или к любой её части. (Для дополнительных разрешений может быть заявлено
+требование об их удалении в определенных случаях, когда Вы вносите изменения в
+произведение.) Вы можете добавлять дополнительные разрешения к добавленным Вами
+в Лицензионное произведение материалам, на которые Вы обладаете авторскими
+правами или правом выдачи соответствующего разрешения.
+
+  Независимо от любых других положений настоящей Лицензии, Вы можете дополнить
+следующими условиями положения настоящей Лицензии в отношении материала,
+добавленного к Лицензионному произведению (если это разрешено обладателями
+авторских прав на материал):
+
+    a) отказом от гарантий или ограничением ответственности, отличающимися от
+    тех, что описаны в разделах 15 и 16 настоящей Лицензии; либо
+
+    b) требованием сохранения соответствующей информации о правах или об
+    авторстве материала, или включения её в Информацию об авторском праве,
+    отображаемую содержащим его произведением; либо
+
+    c) запретом на искажение информации об источнике происхождения материала
+    или требованием того, чтобы измененные версии такого материала содержали
+    корректную отметку об отличиях от исходной версии; либо
+
+    d) ограничением использования в целях рекламы имен лицензиаров или авторов
+    материала; либо
+
+    e) отказом от предоставления прав на использование в качестве товарных
+    знаков некоторых торговых наименований, товарных знаков или знаков
+    обслуживания; либо
+
+    f) требованием от каждого, кто по договору передает материал (или его
+    измененные версии), предоставления компенсации лицензиарам и авторам
+    материала в виде принятия на себя любой ответственности, которую этот
+    договор налагает на лицензиаров и авторов.
+
+  Все остальные ограничительные дополнительные условия считаются "дополнительными
+запретами" по смыслу раздела 10. Если программа, которую Вы получили, или любая
+её часть содержит уведомление о том, что наряду с настоящей Лицензией её
+использование регулируется условием, относящимся к дополнительным запретам, Вы
+можете удалить такое условие. Если лицензия содержит дополнительный запрет, но
+допускает лицензирование на измененных условиях или передачу в соответствии с
+настоящей Лицензией, Вы можете добавить к Лицензионному произведению материал,
+используемый на условиях такой лицензии, в том случае, если дополнительный
+запрет не сохраняется при таком изменении условий лицензии или передаче.
+
+  Если Вы добавляете условия для использования Лицензионного произведения в
+соответствии с настоящим разделом, Вы должны поместить в соответствующих файлах
+исходного текста уведомление о том, что к этим файлам применяются дополнительные
+условия, или указание на то, как ознакомиться с соответствующими условиями.
+
+  Дополнительные разрешающие или ограничивающие условия могут быть сформулированы
+в виде отдельной лицензии или зафиксированы как исключения; вышеуказанные
+требования применяются в любом случае.
+
+  8. Прекращение действия.
+
+  Вы не можете распространять Лицензионное произведение или вносить в него
+изменения на условиях, отличающихся от явно оговоренных в настоящей Лицензии.
+Любая попытка распространения или внесения изменений на иных условиях является
+ничтожной и автоматически прекращает Ваши права, полученные по настоящей
+Лицензии (включая лицензию на любые патенты, предоставленные согласно третьему
+пункту раздела 11).
+
+  Тем не менее если Вы прекращаете нарушение настоящей Лицензии, Ваши права,
+полученные от конкретного правообладателя, восстанавливаются (а) временно, до
+тех пор пока правообладатель явно и окончательно не прекратит действие Ваших
+прав, и (б) навсегда, если правообладатель не уведомит Вас о нарушении с помощью
+надлежащих средств в течение 60 дней после прекращения нарушений.
+
+  Кроме того, Ваши права, полученные от конкретного правообладателя,
+восстанавливаются навсегда, если правообладатель впервые любым подходящим
+способом уведомляет Вас о нарушении настоящей Лицензии на свое произведение (для
+любого произведения) и Вы устраняете нарушение в течение 30 дней после получения
+уведомления.
+
+  Прекращение Ваших прав, описанное в настоящем разделе, не прекращает действие
+лицензий лиц, которые получили от Вас копии произведения или права,
+предоставляемые настоящей Лицензией. Если Ваши права были прекращены навсегда и
+не восстановлены, Вы не можете вновь получить право на тот же материал на
+условиях, описанных в разделе 10.
+
+  9. Акцепт не требуется для получения копий.
+
+  Вы не обязаны принимать условия настоящей Лицензии для того, чтобы получить или
+запустить копию Программы. Случайное распространение Лицензионного произведения,
+происходящее вследствие использования одноранговой (пиринговой) сети для
+получения его копии, также не требует принятия этих условий. Тем не менее только
+настоящая Лицензия дает Вам право распространять или изменять любое Лицензионное
+произведение. Если Вы не приняли условия настоящей Лицензии, такие действия
+будут нарушением авторского права. Поэтому изменяя или распространяя
+Лицензионное произведение, Вы выражаете согласие с условиями настоящей Лицензии.
+
+  10. Автоматическое получение прав последующими получателями.
+
+  Каждый раз, когда Вы передаете Лицензионное произведение, получатель
+автоматически получает от его лицензиара право запускать, изменять и
+распространять это произведение при условии соблюдения настоящей Лицензии. Вы не
+несете ответственности за соблюдение третьими лицами условий настоящей Лицензии.
+
+  "Реорганизацией" называются действия, в результате которых передается управление
+организацией или значительная часть её активов, а также происходит разделение
+или слияние организаций. Если распространение Лицензионного произведения
+является результатом реорганизации, каждая из сторон сделки, получающая копию
+произведения, также получает все права на произведение, которые предшествующее
+юридическое лицо имело или могло предоставить согласно предыдущему абзацу, а
+также право на владение Полным исходным текстом произведения от предшественника,
+осуществляемое в его интересах, если предшественник владеет им или может
+получить его при разумных усилиях.
+
+  Вы не можете налагать каких-либо дополнительных ограничений на осуществление
+прав, предоставленных или подтвержденных в соответствии с настоящей Лицензией.
+Например, Вы не можете ставить осуществление прав, предоставленных по настоящей
+Лицензии, в зависимость от оплаты отчислений, роялти или других сборов; также Вы
+не можете инициировать судебный процесс (включая встречный иск или заявление
+встречного требования в судебном процессе) о нарушении любых патентных прав при
+создании, использовании, продаже, предложении продажи, импорте Программы или
+любой её части.
+
+  11. Патенты.
+
+  "Инвестором" называется правообладатель, разрешающий использование Программы
+либо произведения, на котором основана Программа, на условиях настоящей
+Лицензии. Произведение, лицензированное таким образом, называется "версией со
+вкладом" инвестора.
+
+  "Неотъемлемые патентные претензии" инвестора -- все патентные права,
+принадлежащие инвестору или контролируемые им в настоящее время либо
+приобретенные в будущем, которые могут быть нарушены созданием, использованием
+или продажей версии со вкладом, допускаемыми настоящей Лицензией; они не
+включают в себя права, которые будут нарушены исключительно вследствие будущих
+изменений версии со вкладом. Для целей данного определения под "контролем"
+понимается право выдавать патентные сублицензии способами, не нарушающими
+требований настоящей Лицензии.
+
+  Каждый инвестор предоставляет Вам неисключительную безвозмездную лицензию на
+патент, действующую во всем мире, соответствующую неотъемлемым патентным
+претензиям инвестора, на создание, использование, продажу, предложение для
+продажи, импорт, а также запуск, внесение изменений и распространение всего, что
+входит в состав версии со вкладом.
+
+  В следующих трех абзацах "лицензией на патент" называется любое явно выраженное
+вовне согласие или обязательство не применять патент (например, выдача
+разрешения на использование запатентованного объекта или обещание не подавать в
+суд за нарушение патента). "Выдать" кому-то такую лицензию на патент означает
+заключить такое соглашение или обязаться не применять патент против него.
+
+  Если Вы передаете Лицензионное произведение, сознательно основываясь на лицензии
+на патент, в то время как Полный исходный текст произведения невозможно
+бесплатно скопировать с общедоступного сервера или другим не вызывающим
+затруднений способом, Вы должны либо (1) обеспечить возможность такого доступа к
+Полному исходному тексту, либо (2) отказаться от прав, предоставленных по
+лицензии на патент для данного произведения, либо (3) принять меры по передаче
+лицензии на патент последующим получателям произведения, в соответствии с
+требованиями настоящей Лицензии. "Сознательно основываясь" означает, что Вы
+знаете, что при отсутствии лицензии на патент передача Вами Лицензионного
+произведения в определенной стране или использование получателем переданного ему
+Вами Лицензионного произведения в этой стране нарушит один или несколько
+определенных патентов этой страны, срок действия которых не истек.
+
+  Если в соответствии или в связи с единичной сделкой либо соглашением Вы
+передаете или делаете заказ на распространение Лицензионного произведения, и
+предоставляете определенным лицам, получающим Лицензионное произведение,
+лицензию на патент, разрешающую им использовать, распространять, вносить
+изменения или передавать конкретные экземпляры Лицензионного произведения,
+права, которые Вы предоставляете по лицензии на патент, автоматически переходят
+ко всем получателям Лицензионного произведения и произведений, созданных на его
+основе.
+
+  Патентная лицензия называется "дискриминирующей", если она не покрывает,
+запрещает осуществление или содержит в качестве условия отказ от применения
+одного или нескольких прав, предоставленных настоящей Лицензией. Вы не можете
+передавать Лицензионное произведение, если Вы являетесь участником договора с
+третьим лицом, осуществляющим распространение программного обеспечения, в
+соответствии с которым Вы делаете в пользу третьего лица выплаты, размер которых
+зависит от масштабов Вашей деятельности по передаче произведения, и в
+соответствии с которым любое третье лицо, получающее от Вас Лицензионное
+произведение, делает это на условиях дискриминирующей патентной лицензии (а)
+которая зависит от количества копий Лицензионного произведения, переданных Вами
+(или копий, сделанных с этих копий), или (b) которая используется
+преимущественно в конкретных товарах или подборках, содержащих Лицензионное
+произведение, или в связи с ними, в том случае, если Вы заключили данный договор
+или получили лицензию на патент после 28 марта 2007 года.
+
+  Ничто в настоящей Лицензии не должно толковаться как исключение или ограничение
+любого предполагаемого права или других способов противодействия нарушениям,
+которые во всем остальном могут быть доступны для Вас в соответствии с
+применимым патентным правом.
+
+  12. Запрет отказывать в свободе другим.
+
+  Если на Вас наложены обязанности (будь то по решению суда, договору или иным
+способом), которые противоречат условиям настоящей Лицензии, это не освобождает
+Вас от соблюдения её условий. Если Вы не можете передать Лицензионное
+произведение так, чтобы одновременно выполнять Ваши обязательства по настоящей
+Лицензии и любые другие относящиеся к делу обязательства, то Вы не можете
+передавать его вообще. Например, если Вы согласны с условием, обязывающими Вас
+производить сбор отчислений за дальнейшую передачу от тех, кому Вы передаете
+Программу, то для того, чтобы соблюсти это условие и выполнить требования
+настоящей Лицензии, Вы должны полностью воздержаться от передачи Программы.
+
+  13. Удаленное сетевое взаимодействие.
+
+  Под "Прокси-программой" понимается отдельная программа, специально
+разработанная для использования совместно с Лицензионным произведением,
+и взаимодействующая с ним прямо или косвенно через любой вид программного
+интерфейса, компьютерную сеть, имитацию такой сети, или, в свою очередь,
+через другую Прокси-программу.
+
+  Независимо от любых других положений настоящей Лицензии, если вы
+предоставляете любому пользователю возможность взаимодействовать с Лицензионным
+произведением через компьютерную сеть, имитацию такой сети, или через любое
+количество "Прокси-программ", вы должны в явной форме предложить этому
+пользователю возможность получить Полный исходный текст Лицензионного
+произведения и всех Прокси-программ путём предоставления доступа к нему
+с сетевого сервера без взимания платы, посредством стандартных или
+традиционных способов, используемых для копирования программного обеспечения.
+Полный исходный текст Лицензионного произведения должен предоставляться
+пользователю на условиях настоящей Лицензии, а Полный исходный текст
+Прокси-программ должен предоставляться пользователю либо на условиях настоящей
+Лицензии, либо на условиях одной из свободных лицензий, совместимых с
+Генеральной публичной Лицензией GNU, перечисленных Фондом Свободного
+Программного Обеспечения в списке под названием "Лицензии свободных программ,
+совместимые с GPL".
+
+  14. Пересмотренные редакции настоящей Лицензии.
+
+  Автор настоящей Лицензии время от времени может публиковать пересмотренные
+и (или) новые редакции Сетевой Публичной Лицензии Vitastor. Они будут аналогичны
+по смыслу настоящей редакции, но могут отличаться от нее в деталях, направленных
+на решение новых проблем или регулирование новых отношений.
+
+  Каждой редакции присваивается собственный номер. Если для Программы указано,
+что к ней применима определенная редакция Сетевой Публичной Лицензии Vitastor
+"или любая более поздняя редакция", у Вас есть возможность использовать термины
+и условия, содержащиеся в редакции с указанным номером или любой более поздней
+редакции, опубликованной автором настоящей Лицензии. Если для Программы не
+указан номер редакции Сетевой Публичной Лицензии Vitastor, Вы можете выбрать
+любую редакцию, опубликованную автором настоящей Лицензии.
+
+  Более поздние редакции Лицензии могут дать Вам дополнительные или принципиально
+иные права. Тем не менее в результате Вашего выбора более поздней редакции на
+автора или правообладателя не возлагается никаких дополнительных обязанностей.
+
+  15. Отказ от гарантий.
+
+  НА ПРОГРАММУ НЕ ПРЕДОСТАВЛЯЕТСЯ НИКАКИХ ГАРАНТИЙ ЗА ИСКЛЮЧЕНИЕМ ПРЕДУСМОТРЕННЫХ
+ДЕЙСТВУЮЩИМ ЗАКОНОДАТЕЛЬСТВОМ. ЕСЛИ ИНОЕ НЕ УКАЗАНО В ПИСЬМЕННОЙ ФОРМЕ,
+ПРАВООБЛАДАТЕЛИ И (ИЛИ) ТРЕТЬИ ЛИЦА ПРЕДОСТАВЛЯЮТ ПРОГРАММУ "КАК ЕСТЬ", БЕЗ
+КАКИХ-ЛИБО ЯВНЫХ ИЛИ ПОДРАЗУМЕВАЕМЫХ ГАРАНТИЙ, ВКЛЮЧАЯ ГАРАНТИИ ПРИГОДНОСТИ ДЛЯ
+КОНКРЕТНЫХ ЦЕЛЕЙ, НО НЕ ОГРАНИЧИВАЯСЬ ИМИ. ВЕСЬ РИСК, СВЯЗАННЫЙ С КАЧЕСТВОМ И
+ПРОИЗВОДИТЕЛЬНОСТЬЮ ПРОГРАММЫ, ВОЗЛАГАЕТСЯ НА ВАС. ЕСЛИ В ПРОГРАММЕ БУДУТ
+ВЫЯВЛЕНЫ НЕДОСТАТКИ, ВЫ ПРИНИМАЕТЕ НА СЕБЯ СТОИМОСТЬ ВСЕГО НЕОБХОДИМОГО
+ОБСЛУЖИВАНИЯ, РЕМОНТА ИЛИ ИСПРАВЛЕНИЯ.
+
+  16. Ограничение ответственности.
+
+  ЕСЛИ ИНОЕ НЕ ПРЕДУСМОТРЕНО ДЕЙСТВУЮЩИМ ЗАКОНОДАТЕЛЬСТВОМ ИЛИ СОГЛАШЕНИЕМ СТОРОН,
+ЗАКЛЮЧЕННЫМ В ПИСЬМЕННОЙ ФОРМЕ, ПРАВООБЛАДАТЕЛЬ ИЛИ ИНОЕ ЛИЦО, КОТОРОЕ ВНОСИТ
+ИЗМЕНЕНИЯ В ПРОГРАММУ И (ИЛИ) ПЕРЕДАЕТ ЕЁ НА УСЛОВИЯХ, СФОРМУЛИРОВАННЫХ ВЫШЕ, НЕ
+МОЖЕТ НЕСТИ ОТВЕТСТВЕННОСТЬ ПЕРЕД ВАМИ ЗА ПРИЧИНЕННЫЙ УЩЕРБ, ВКЛЮЧАЯ УЩЕРБ
+ОБЩЕГО ЛИБО КОНКРЕТНОГО ХАРАКТЕРА, ПРИЧИНЕННЫЙ СЛУЧАЙНО ИЛИ ЯВЛЯЮЩИЙСЯ
+СЛЕДСТВИЕМ ИСПОЛЬЗОВАНИЯ ПРОГРАММЫ ЛИБО НЕВОЗМОЖНОСТИ ЕЁ ИСПОЛЬЗОВАНИЯ (В ТОМ
+ЧИСЛЕ ЗА УНИЧТОЖЕНИЕ ИЛИ МОДИФИКАЦИЮ ИНФОРМАЦИИ, ЛИБО УБЫТКИ, ПОНЕСЕННЫЕ ВАМИ
+ИЛИ ТРЕТЬИМИ ЛИЦАМИ, ЛИБО СБОИ ПРОГРАММЫ ПРИ ВЗАИМОДЕЙСТВИИ С ДРУГИМ ПРОГРАММНЫМ
+ОБЕСПЕЧЕНИЕМ), В ТОМ ЧИСЛЕ И В СЛУЧАЯХ, КОГДА ПРАВООБЛАДАТЕЛЬ ИЛИ ТРЕТЬЕ ЛИЦО
+ПРЕДУПРЕЖДЕНЫ О ВОЗМОЖНОСТИ ПРИЧИНЕНИЯ ТАКИХ УБЫТКОВ.
+
+  17. Толкование разделов 15 и 16.
+
+  Если отказ от гарантии и ограничение ответственности, представленные выше, по
+закону не могут быть применены в соответствии с их условиями, суды,
+рассматривающие спор, должны применить действующий закон, который в наибольшей
+степени предусматривает абсолютный отказ от всей гражданской ответственности в
+связи с Программой, за исключением случаев, когда гарантия или принятие на себя
+ответственности за копию программы предоставляется за плату.
+
+                        КОНЕЦ ОПРЕДЕЛЕНИЙ И УСЛОВИЙ
+
+          Порядок применения условий Лицензии к Вашим программам
+
+  Если Вы разрабатываете новую программу и хотите, чтобы её использование принесло
+максимальную пользу обществу, наилучший способ достичь этого -- сделать её
+свободной, чтобы все могли распространять и изменять её на условиях настоящей
+Лицензии.
+
+  Для этого сделайте так, чтобы программа содержала в себе описанные ниже
+уведомления. Самым надежным способом это сделать является включение их в начало
+каждого файла исходного текста, чтобы наиболее эффективным образом сообщить об
+отсутствии гарантий; каждый файл должен иметь по меньшей мере одну строку с
+оповещением об авторских правах и указанием на то, где находится полный текст
+уведомлений.
+
+    <Строка с названием Программы и информацией о её назначении.>
+    Copyright © <год выпуска программы в свет>  <имя автора>
+
+    Эта программа является свободным программным обеспечением: Вы можете
+    распространять её и (или) изменять, соблюдая условия Сетевой Публичной
+    Лицензии Vitastor, опубликованной автором Vitastor, либо редакции 1.1
+    Лицензии, либо (на Ваше усмотрение) любой редакции, выпущенной позже.
+
+    Эта программа распространяется в расчете на то, что она окажется полезной,
+    но БЕЗ КАКИХ-ЛИБО ГАРАНТИЙ, включая подразумеваемую гарантию КАЧЕСТВА либо
+    ПРИГОДНОСТИ ДЛЯ ОПРЕДЕЛЕННЫХ ЦЕЛЕЙ. Ознакомьтесь с Сетевой Публичной
+    Лицензией Vitastor для получения более подробной информации.
+
+  Также добавьте информацию о том, как связаться с Вами посредством электронной
+или обычной почты.
+
+  Если ваша программа взаимодействует с пользователями удаленно через
+компьютерную сеть, Вы также должны убедиться, что обеспечили её пользователям
+возможность получить её исходные тексты. Например, если Ваша программа является
+веб-приложением, её интерфейс может отображать ссылку "Исходные коды", которая
+указывает на архив с текстом. Существует много способов, которыми Вы можете
+распространять исходные тексты, для разных программ подходят разные решения;
+ознакомьтесь с разделом 13 для того, чтобы узнать конкретные требования.

VNPL-1.1.txt

@@ -0,0 +1,648 @@
+                     VITASTOR NETWORK PUBLIC LICENSE
+                     Version 1.1,  6 February 2021
+
+ Copyright (C) 2021 Vitaliy Filippov <vitalif@yourcmc.ru>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The Vitastor Network Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+GNU General Public Licenses and Vitastor Network Public License are
+intended to guarantee your freedom to share and change all versions
+of a program--to make sure it remains free software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  GNU General Public Licenses and Vitastor Network Public License
+are designed to make sure that you have the freedom to distribute copies
+of free software (and charge for them if you wish), that you receive
+source code or can get it if you want it, that you can change the software
+or use pieces of it in new free programs, and that you know you can do these
+things.
+
+  Developers that use GNU General Public Licenses and Vitastor
+Network Public License protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public. Even the GNU Affero General Public License
+permits running a modified version in a closed environment where
+public users only interact with it through a closed-source proxy, again,
+without making the program and the proxy available to the public
+for free.
+
+  The Vitastor Network Public License is designed specifically to
+ensure that, in such cases, the modified program and the proxy stays
+available to the community. It requires the operator of a network server to
+provide the source code of the original program and all other programs
+communicating with it running there to the users of that server.
+Therefore, public use of a modified version, on a server accessible
+directly or indirectly to the public, gives the public access to the source
+code of the modified version.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 1.1 of the Vitastor Network Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction.
+
+  A "Proxy Program" means a separate program which is specially designed to
+be used in conjunction with the covered work and interacts with it directly
+or indirectly through any kind of API (application programming interfaces),
+a computer network, an imitation of such network, or another Proxy Program
+itself.
+
+  Notwithstanding any other provision of this License, if you provide any user
+with an opportunity to interact with the covered work through a computer
+network, an imitation of such network, or any number of "Proxy Programs",
+you must prominently offer that user an opportunity to receive the
+Corresponding Source of the covered work and all Proxy Programs from a
+network server at no charge, through some standard or customary means of
+facilitating copying of software. The Corresponding Source for the covered
+work must be made available under the conditions of this License, and
+the Corresponding Source for all Proxy Programs must be made available
+under the conditions of either this License or any GPL-Compatible
+Free Software License, as described by the Free Software Foundation
+in their "GPL-Compatible License List".
+
+  14. Revised Versions of this License.
+
+  Vitastor Author may publish revised and/or new versions of
+the Vitastor Network Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the Vitastor Network
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version. If the Program does not specify a version
+number of the Vitastor Network Public License, you may choose any version
+ever published.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the Vitastor Network Public License as published by
+    the Vitastor Author, either version 1.1 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    Vitastor Network Public License for more details.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.

allocator.cpp

@@ -1,127 +0,0 @@
-#include <stdexcept>
-#include "allocator.h"
-
-#include <stdlib.h>
-#include <malloc.h>
-
-allocator::allocator(uint64_t blocks)
-{
-    if (blocks >= 0x80000000 || blocks <= 1)
-    {
-        throw std::invalid_argument("blocks");
-    }
-    uint64_t p2 = 1, total = 1;
-    while (p2 * 64 < blocks)
-    {
-        p2 = p2 * 64;
-        total += p2;
-    }
-    total -= p2;
-    total += (blocks+63) / 64;
-    mask = new uint64_t[2 + total];
-    size = free = blocks;
-    last_one_mask = (blocks % 64) == 0
-        ? UINT64_MAX
-        : ~(UINT64_MAX << (64 - blocks % 64));
-    for (uint64_t i = 0; i < total; i++)
-    {
-        mask[i] = 0;
-    }
-}
-
-allocator::~allocator()
-{
-    delete[] mask;
-}
-
-void allocator::set(uint64_t addr, bool value)
-{
-    if (addr >= size)
-    {
-        return;
-    }
-    uint64_t p2 = 1, offset = 0;
-    while (p2 * 64 < size)
-    {
-        offset += p2;
-        p2 = p2 * 64;
-    }
-    uint64_t cur_addr = addr;
-    bool is_last = true;
-    uint64_t value64 = value ? 1 : 0;
-    while (1)
-    {
-        uint64_t last = offset + cur_addr/64;
-        uint64_t bit = cur_addr % 64;
-        if (((mask[last] >> bit) & 1) != value64)
-        {
-            if (is_last)
-            {
-                free += value ? -1 : 1;
-            }
-            if (value)
-            {
-                mask[last] = mask[last] | (1l << bit);
-                if (mask[last] != (!is_last || cur_addr/64 < size/64
-                    ? UINT64_MAX : last_one_mask))
-                {
-                    break;
-                }
-            }
-            else
-            {
-                mask[last] = mask[last] & ~(1l << bit);
-            }
-            is_last = false;
-            if (p2 > 1)
-            {
-                p2 = p2 / 64;
-                offset -= p2;
-                cur_addr /= 64;
-            }
-            else
-            {
-                break;
-            }
-        }
-        else
-        {
-            break;
-        }
-    }
-}
-
-uint64_t allocator::find_free()
-{
-    uint64_t p2 = 1, offset = 0, addr = 0, f, i;
-    while (p2 < size)
-    {
-        uint64_t m = mask[offset + addr];
-        for (i = 0, f = 1; i < 64; i++, f <<= 1)
-        {
-            if (!(m & f))
-            {
-                break;
-            }
-        }
-        if (i == 64)
-        {
-            // No space
-            return UINT64_MAX;
-        }
-        addr = (addr * 64) | i;
-        if (addr >= size)
-        {
-            // No space
-            return UINT64_MAX;
-        }
-        offset += p2;
-        p2 = p2 * 64;
-    }
-    return addr;
-}
-
-uint64_t allocator::get_free_count()
-{
-    return free;
-}

allocator.h

@@ -1,18 +0,0 @@
-#pragma once
-
-#include <stdint.h>
-
-// Hierarchical bitmap allocator
-class allocator
-{
-    uint64_t size;
-    uint64_t free;
-    uint64_t last_one_mask;
-    uint64_t *mask;
-public:
-    allocator(uint64_t blocks);
-    ~allocator();
-    void set(uint64_t addr, bool value);
-    uint64_t find_free();
-    uint64_t get_free_count();
-};

blockstore.cpp

@@ -1,61 +0,0 @@
-#include "blockstore_impl.h"
-
-blockstore_t::blockstore_t(blockstore_config_t & config, ring_loop_t *ringloop)
-{
-    impl = new blockstore_impl_t(config, ringloop);
-}
-
-blockstore_t::~blockstore_t()
-{
-    delete impl;
-}
-
-void blockstore_t::loop()
-{
-    impl->loop();
-}
-
-bool blockstore_t::is_started()
-{
-    return impl->is_started();
-}
-
-bool blockstore_t::is_stalled()
-{
-    return impl->is_stalled();
-}
-
-bool blockstore_t::is_safe_to_stop()
-{
-    return impl->is_safe_to_stop();
-}
-
-void blockstore_t::enqueue_op(blockstore_op_t *op)
-{
-    impl->enqueue_op(op, false);
-}
-
-void blockstore_t::enqueue_op_first(blockstore_op_t *op)
-{
-    impl->enqueue_op(op, true);
-}
-
-std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
-{
-    return impl->unstable_writes;
-}
-
-uint32_t blockstore_t::get_block_size()
-{
-    return impl->get_block_size();
-}
-
-uint64_t blockstore_t::get_block_count()
-{
-    return impl->get_block_count();
-}
-
-uint32_t blockstore_t::get_disk_alignment()
-{
-    return impl->get_disk_alignment();
-}

blockstore_flush.cpp

@@ -1,700 +0,0 @@
-#include "blockstore_impl.h"
-
-journal_flusher_t::journal_flusher_t(int flusher_count, blockstore_impl_t *bs)
-{
-    this->bs = bs;
-    this->flusher_count = flusher_count;
-    active_flushers = 0;
-    sync_threshold = flusher_count == 1 ? 1 : flusher_count/2;
-    journal_trim_interval = sync_threshold;
-    journal_trim_counter = 0;
-    journal_superblock = bs->journal.inmemory ? bs->journal.buffer : memalign(MEM_ALIGNMENT, bs->journal_block_size);
-    co = new journal_flusher_co[flusher_count];
-    for (int i = 0; i < flusher_count; i++)
-    {
-        co[i].bs = bs;
-        co[i].flusher = this;
-    }
-}
-
-journal_flusher_co::journal_flusher_co()
-{
-    wait_state = 0;
-    simple_callback_r = [this](ring_data_t* data)
-    {
-        bs->live = true;
-        if (data->res != data->iov.iov_len)
-        {
-            throw std::runtime_error(
-                "data read operation failed during flush ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-                "). can't continue, sorry :-("
-            );
-        }
-        wait_count--;
-    };
-    simple_callback_w = [this](ring_data_t* data)
-    {
-        bs->live = true;
-        if (data->res != data->iov.iov_len)
-        {
-            throw std::runtime_error(
-                "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-                "). state "+std::to_string(wait_state)+". in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-            );
-        }
-        wait_count--;
-    };
-}
-
-journal_flusher_t::~journal_flusher_t()
-{
-    if (!bs->journal.inmemory)
-        free(journal_superblock);
-    delete[] co;
-}
-
-bool journal_flusher_t::is_active()
-{
-    return active_flushers > 0 || start_forced && flush_queue.size() > 0 || flush_queue.size() >= sync_threshold;
-}
-
-void journal_flusher_t::loop()
-{
-    for (int i = 0; i < flusher_count; i++)
-    {
-        if (!active_flushers && (start_forced ? !flush_queue.size() : (flush_queue.size() < sync_threshold)))
-        {
-            return;
-        }
-        co[i].loop();
-    }
-}
-
-void journal_flusher_t::enqueue_flush(obj_ver_id ov)
-{
-    auto it = flush_versions.find(ov.oid);
-    if (it != flush_versions.end())
-    {
-        if (it->second < ov.version)
-            it->second = ov.version;
-    }
-    else
-    {
-        flush_versions[ov.oid] = ov.version;
-        flush_queue.push_back(ov.oid);
-    }
-}
-
-void journal_flusher_t::unshift_flush(obj_ver_id ov)
-{
-    auto it = flush_versions.find(ov.oid);
-    if (it != flush_versions.end())
-    {
-        if (it->second < ov.version)
-            it->second = ov.version;
-    }
-    else
-    {
-        flush_versions[ov.oid] = ov.version;
-        flush_queue.push_front(ov.oid);
-    }
-}
-
-void journal_flusher_t::force_start()
-{
-    start_forced = true;
-    bs->ringloop->wakeup();
-}
-
-#define await_sqe(label) \
-    resume_##label:\
-        sqe = bs->get_sqe();\
-        if (!sqe)\
-        {\
-            wait_state = label;\
-            return false;\
-        }\
-        data = ((ring_data_t*)sqe->user_data);
-
-bool journal_flusher_co::loop()
-{
-    // This is much better than implementing the whole function as an FSM
-    // Maybe I should consider a coroutine library like https://github.com/hnes/libaco ...
-    if (wait_state == 1)
-        goto resume_1;
-    else if (wait_state == 2)
-        goto resume_2;
-    else if (wait_state == 3)
-        goto resume_3;
-    else if (wait_state == 4)
-        goto resume_4;
-    else if (wait_state == 5)
-        goto resume_5;
-    else if (wait_state == 6)
-        goto resume_6;
-    else if (wait_state == 7)
-        goto resume_7;
-    else if (wait_state == 8)
-        goto resume_8;
-    else if (wait_state == 9)
-        goto resume_9;
-    else if (wait_state == 10)
-        goto resume_10;
-    else if (wait_state == 12)
-        goto resume_12;
-    else if (wait_state == 13)
-        goto resume_13;
-    else if (wait_state == 14)
-        goto resume_14;
-    else if (wait_state == 15)
-        goto resume_15;
-    else if (wait_state == 16)
-        goto resume_16;
-    else if (wait_state == 17)
-        goto resume_17;
-    else if (wait_state == 18)
-        goto resume_18;
-resume_0:
-    if (!flusher->flush_queue.size() ||
-        !flusher->start_forced && !flusher->active_flushers && flusher->flush_queue.size() < flusher->sync_threshold)
-    {
-        flusher->start_forced = false;
-        wait_state = 0;
-        return true;
-    }
-    cur.oid = flusher->flush_queue.front();
-    cur.version = flusher->flush_versions[cur.oid];
-    flusher->flush_queue.pop_front();
-    flusher->flush_versions.erase(cur.oid);
-    dirty_end = bs->dirty_db.find(cur);
-    if (dirty_end != bs->dirty_db.end())
-    {
-        repeat_it = flusher->sync_to_repeat.find(cur.oid);
-        if (repeat_it != flusher->sync_to_repeat.end())
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Postpone %lu:%lu v%lu\n", cur.oid.inode, cur.oid.stripe, cur.version);
-#endif
-            // We don't flush different parts of history of the same object in parallel
-            // So we check if someone is already flushing this object
-            // In that case we set sync_to_repeat and pick another object
-            // Another coroutine will see it and re-queue the object after it finishes
-            if (repeat_it->second < cur.version)
-                repeat_it->second = cur.version;
-            wait_state = 0;
-            goto resume_0;
-        }
-        else
-            flusher->sync_to_repeat[cur.oid] = 0;
-#ifdef BLOCKSTORE_DEBUG
-        printf("Flushing %lu:%lu v%lu\n", cur.oid.inode, cur.oid.stripe, cur.version);
-#endif
-        flusher->active_flushers++;
-resume_1:
-        // Scan dirty versions of the object
-        if (!scan_dirty(1))
-        {
-            wait_state += 1;
-            return false;
-        }
-        if (copy_count == 0 && clean_loc == UINT64_MAX && !has_delete && !has_empty)
-        {
-            // Nothing to flush
-            flusher->active_flushers--;
-            repeat_it = flusher->sync_to_repeat.find(cur.oid);
-            if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version)
-            {
-                // Requeue version
-                flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second });
-            }
-            flusher->sync_to_repeat.erase(repeat_it);
-            wait_state = 0;
-            goto resume_0;
-        }
-        // Find it in clean_db
-        clean_it = bs->clean_db.find(cur.oid);
-        old_clean_loc = (clean_it != bs->clean_db.end() ? clean_it->second.location : UINT64_MAX);
-        if (clean_loc == UINT64_MAX)
-        {
-            if (copy_count > 0 && has_delete || old_clean_loc == UINT64_MAX)
-            {
-                // Object not allocated. This is a bug.
-                char err[1024];
-                snprintf(
-                    err, 1024, "BUG: Object %lu:%lu v%lu that we are trying to flush is not allocated on the data device",
-                    cur.oid.inode, cur.oid.stripe, cur.version
-                );
-                throw std::runtime_error(err);
-            }
-            else
-            {
-                clean_loc = old_clean_loc;
-            }
-        }
-        // Also we need to submit metadata read(s). We do read-modify-write cycle(s) for every operation.
-    resume_2:
-        if (!modify_meta_read(clean_loc, meta_new, 2))
-        {
-            wait_state += 2;
-            return false;
-        }
-        if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc)
-        {
-        resume_14:
-            if (!modify_meta_read(old_clean_loc, meta_old, 14))
-            {
-                wait_state += 14;
-                return false;
-            }
-        }
-        else
-            meta_old.submitted = false;
-    resume_3:
-        if (wait_count > 0)
-        {
-            wait_state = 3;
-            return false;
-        }
-        if (meta_new.submitted)
-        {
-            meta_new.it->second.state = 1;
-            bs->ringloop->wakeup();
-        }
-        if (meta_old.submitted)
-        {
-            meta_old.it->second.state = 1;
-            bs->ringloop->wakeup();
-        }
-        // Reads completed, submit writes and set bitmap bits
-        if (bs->clean_entry_bitmap_size)
-        {
-            new_clean_bitmap = (bs->inmemory_meta
-                ? meta_new.buf + meta_new.pos*bs->clean_entry_size + sizeof(clean_disk_entry)
-                : bs->clean_bitmap + (clean_loc >> bs->block_order)*bs->clean_entry_bitmap_size);
-            if (clean_init_bitmap)
-            {
-                memset(new_clean_bitmap, 0, bs->clean_entry_bitmap_size);
-                bitmap_set(new_clean_bitmap, clean_bitmap_offset, clean_bitmap_len);
-            }
-        }
-        for (it = v.begin(); it != v.end(); it++)
-        {
-            if (new_clean_bitmap)
-            {
-                bitmap_set(new_clean_bitmap, it->offset, it->len);
-            }
-            await_sqe(4);
-            data->iov = (struct iovec){ it->buf, (size_t)it->len };
-            data->callback = simple_callback_w;
-            my_uring_prep_writev(
-                sqe, bs->data_fd, &data->iov, 1, bs->data_offset + clean_loc + it->offset
-            );
-            wait_count++;
-        }
-        // Sync data before writing metadata
-    resume_16:
-    resume_17:
-    resume_18:
-        if (copy_count && !fsync_batch(false, 16))
-        {
-            wait_state += 16;
-            return false;
-        }
-    resume_5:
-        // And metadata writes, but only after data writes complete
-        if (!bs->inmemory_meta && meta_new.it->second.state == 0 || wait_count > 0)
-        {
-            // metadata sector is still being read or data is still being written, wait for it
-            wait_state = 5;
-            return false;
-        }
-        if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc)
-        {
-            if (!bs->inmemory_meta && meta_old.it->second.state == 0)
-            {
-                wait_state = 5;
-                return false;
-            }
-            memset(meta_old.buf + meta_old.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
-            await_sqe(15);
-            data->iov = (struct iovec){ meta_old.buf, bs->meta_block_size };
-            data->callback = simple_callback_w;
-            my_uring_prep_writev(
-                sqe, bs->meta_fd, &data->iov, 1, bs->meta_offset + meta_old.sector
-            );
-            wait_count++;
-        }
-        if (has_delete)
-        {
-            memset(meta_new.buf + meta_new.pos*bs->clean_entry_size, 0, bs->clean_entry_size);
-        }
-        else
-        {
-            clean_disk_entry *new_entry = (clean_disk_entry*)(meta_new.buf + meta_new.pos*bs->clean_entry_size);
-            new_entry->oid = cur.oid;
-            new_entry->version = cur.version;
-            if (!bs->inmemory_meta)
-            {
-                memcpy(&new_entry->bitmap, new_clean_bitmap, bs->clean_entry_bitmap_size);
-            }
-        }
-        await_sqe(6);
-        data->iov = (struct iovec){ meta_new.buf, bs->meta_block_size };
-        data->callback = simple_callback_w;
-        my_uring_prep_writev(
-            sqe, bs->meta_fd, &data->iov, 1, bs->meta_offset + meta_new.sector
-        );
-        wait_count++;
-    resume_7:
-        if (wait_count > 0)
-        {
-            wait_state = 7;
-            return false;
-        }
-        // Done, free all buffers
-        if (!bs->inmemory_meta)
-        {
-            meta_new.it->second.usage_count--;
-            if (meta_new.it->second.usage_count == 0)
-            {
-                free(meta_new.it->second.buf);
-                flusher->meta_sectors.erase(meta_new.it);
-            }
-            if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc)
-            {
-                meta_old.it->second.usage_count--;
-                if (meta_old.it->second.usage_count == 0)
-                {
-                    free(meta_old.it->second.buf);
-                    flusher->meta_sectors.erase(meta_old.it);
-                }
-            }
-        }
-        for (it = v.begin(); it != v.end(); it++)
-        {
-            free(it->buf);
-        }
-        v.clear();
-        // And sync metadata (in batches - not per each operation!)
-    resume_8:
-    resume_9:
-    resume_10:
-        if (!fsync_batch(true, 8))
-        {
-            wait_state += 8;
-            return false;
-        }
-        // Update clean_db and dirty_db, free old data locations
-        update_clean_db();
-        // Clear unused part of the journal every <journal_trim_interval> flushes
-        if (!((++flusher->journal_trim_counter) % flusher->journal_trim_interval))
-        {
-            flusher->journal_trim_counter = 0;
-            if (bs->journal.trim())
-            {
-                // Update journal "superblock"
-                await_sqe(12);
-                *((journal_entry_start*)flusher->journal_superblock) = {
-                    .crc32 = 0,
-                    .magic = JOURNAL_MAGIC,
-                    .type = JE_START,
-                    .size = sizeof(journal_entry_start),
-                    .reserved = 0,
-                    .journal_start = bs->journal.used_start,
-                };
-                ((journal_entry_start*)flusher->journal_superblock)->crc32 = je_crc32((journal_entry*)flusher->journal_superblock);
-                data->iov = (struct iovec){ flusher->journal_superblock, bs->journal_block_size };
-                data->callback = simple_callback_w;
-                my_uring_prep_writev(sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset);
-                wait_count++;
-            resume_13:
-                if (wait_count > 0)
-                {
-                    wait_state = 13;
-                    return false;
-                }
-            }
-        }
-        // All done
-#ifdef BLOCKSTORE_DEBUG
-        printf("Flushed %lu:%lu v%lu\n", cur.oid.inode, cur.oid.stripe, cur.version);
-#endif
-        flusher->active_flushers--;
-        repeat_it = flusher->sync_to_repeat.find(cur.oid);
-        if (repeat_it != flusher->sync_to_repeat.end() && repeat_it->second > cur.version)
-        {
-            // Requeue version
-            flusher->unshift_flush({ .oid = cur.oid, .version = repeat_it->second });
-        }
-        flusher->sync_to_repeat.erase(repeat_it);
-        wait_state = 0;
-        goto resume_0;
-    }
-    return true;
-}
-
-bool journal_flusher_co::scan_dirty(int wait_base)
-{
-    if (wait_state == wait_base)
-    {
-        goto resume_0;
-    }
-    dirty_it = dirty_start = dirty_end;
-    v.clear();
-    wait_count = 0;
-    copy_count = 0;
-    clean_loc = UINT64_MAX;
-    has_delete = false;
-    has_empty = false;
-    skip_copy = false;
-    clean_init_bitmap = false;
-    while (1)
-    {
-        if (dirty_it->second.state == ST_J_STABLE && !skip_copy)
-        {
-            // First we submit all reads
-            if (dirty_it->second.len == 0)
-            {
-                has_empty = true;
-            }
-            else
-            {
-                offset = dirty_it->second.offset;
-                end_offset = dirty_it->second.offset + dirty_it->second.len;
-                it = v.begin();
-                while (1)
-                {
-                    for (; it != v.end(); it++)
-                        if (it->offset >= offset)
-                            break;
-                    if (it == v.end() || it->offset > offset && it->len > 0)
-                    {
-                        submit_offset = dirty_it->second.location + offset - dirty_it->second.offset;
-                        submit_len = it == v.end() || it->offset >= end_offset ? end_offset-offset : it->offset-offset;
-                        it = v.insert(it, (copy_buffer_t){ .offset = offset, .len = submit_len, .buf = memalign(MEM_ALIGNMENT, submit_len) });
-                        copy_count++;
-                        if (bs->journal.inmemory)
-                        {
-                            // Take it from memory
-                            memcpy(v.back().buf, bs->journal.buffer + submit_offset, submit_len);
-                        }
-                        else
-                        {
-                            // Read it from disk
-                            await_sqe(0);
-                            data->iov = (struct iovec){ v.back().buf, (size_t)submit_len };
-                            data->callback = simple_callback_r;
-                            my_uring_prep_readv(
-                                sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset + submit_offset
-                            );
-                            wait_count++;
-                        }
-                    }
-                    offset = it->offset+it->len;
-                    if (it == v.end() || offset >= end_offset)
-                        break;
-                }
-            }
-        }
-        else if (dirty_it->second.state == ST_D_STABLE && !skip_copy)
-        {
-            // There is an unflushed big write. Copy small writes in its position
-            clean_loc = dirty_it->second.location;
-            clean_init_bitmap = true;
-            clean_bitmap_offset = dirty_it->second.offset;
-            clean_bitmap_len = dirty_it->second.len;
-            skip_copy = true;
-        }
-        else if (dirty_it->second.state == ST_DEL_STABLE && !skip_copy)
-        {
-            // There is an unflushed delete
-            has_delete = true;
-            skip_copy = true;
-        }
-        else if (!IS_STABLE(dirty_it->second.state))
-        {
-            char err[1024];
-            snprintf(
-                err, 1024, "BUG: Unexpected dirty_entry %lu:%lu v%lu state during flush: %d",
-                dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version, dirty_it->second.state
-            );
-            throw std::runtime_error(err);
-        }
-        dirty_start = dirty_it;
-        if (dirty_it == bs->dirty_db.begin())
-        {
-            break;
-        }
-        dirty_it--;
-        if (dirty_it->first.oid != cur.oid)
-        {
-            break;
-        }
-    }
-    return true;
-}
-
-bool journal_flusher_co::modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base)
-{
-    if (wait_state == wait_base)
-    {
-        goto resume_0;
-    }
-    // We must check if the same sector is already in memory if we don't keep all metadata in memory all the time.
-    // And yet another option is to use LSM trees for metadata, but it sophisticates everything a lot,
-    // so I'll avoid it as long as I can.
-    wr.submitted = false;
-    wr.sector = ((meta_loc >> bs->block_order) / (bs->meta_block_size / bs->clean_entry_size)) * bs->meta_block_size;
-    wr.pos = ((meta_loc >> bs->block_order) % (bs->meta_block_size / bs->clean_entry_size));
-    if (bs->inmemory_meta)
-    {
-        wr.buf = bs->metadata_buffer + wr.sector;
-        return true;
-    }
-    wr.it = flusher->meta_sectors.find(wr.sector);
-    if (wr.it == flusher->meta_sectors.end())
-    {
-        // Not in memory yet, read it
-        wr.buf = memalign(MEM_ALIGNMENT, bs->meta_block_size);
-        wr.it = flusher->meta_sectors.emplace(wr.sector, (meta_sector_t){
-            .offset = wr.sector,
-            .len = bs->meta_block_size,
-            .state = 0, // 0 = not read yet
-            .buf = wr.buf,
-            .usage_count = 1,
-        }).first;
-        await_sqe(0);
-        data->iov = (struct iovec){ wr.it->second.buf, bs->meta_block_size };
-        data->callback = simple_callback_r;
-        wr.submitted = true;
-        my_uring_prep_readv(
-            sqe, bs->meta_fd, &data->iov, 1, bs->meta_offset + wr.sector
-        );
-        wait_count++;
-    }
-    else
-    {
-        wr.buf = wr.it->second.buf;
-        wr.it->second.usage_count++;
-    }
-    return true;
-}
-
-void journal_flusher_co::update_clean_db()
-{
-    if (old_clean_loc != UINT64_MAX && old_clean_loc != clean_loc)
-    {
-#ifdef BLOCKSTORE_DEBUG
-        printf("Free block %lu\n", old_clean_loc >> bs->block_order);
-#endif
-        bs->data_alloc->set(old_clean_loc >> bs->block_order, false);
-    }
-    if (has_delete)
-    {
-        auto clean_it = bs->clean_db.find(cur.oid);
-        bs->clean_db.erase(clean_it);
-        bs->data_alloc->set(clean_loc >> bs->block_order, false);
-        clean_loc = UINT64_MAX;
-    }
-    else
-    {
-        bs->clean_db[cur.oid] = {
-            .version = cur.version,
-            .location = clean_loc,
-        };
-    }
-    bs->erase_dirty(dirty_start, std::next(dirty_end), clean_loc);
-}
-
-bool journal_flusher_co::fsync_batch(bool fsync_meta, int wait_base)
-{
-    if (wait_state == wait_base)
-        goto resume_0;
-    else if (wait_state == wait_base+1)
-        goto resume_1;
-    else if (wait_state == wait_base+2)
-        goto resume_2;
-    if (!(fsync_meta ? bs->disable_meta_fsync : bs->disable_journal_fsync))
-    {
-        cur_sync = flusher->syncs.end();
-        while (cur_sync != flusher->syncs.begin())
-        {
-            cur_sync--;
-            if (cur_sync->fsync_meta == fsync_meta && cur_sync->state == 0)
-            {
-                goto sync_found;
-            }
-        }
-        cur_sync = flusher->syncs.emplace(flusher->syncs.end(), (flusher_sync_t){
-            .fsync_meta = fsync_meta,
-            .ready_count = 0,
-            .state = 0,
-        });
-    sync_found:
-        cur_sync->ready_count++;
-        if (cur_sync->ready_count >= flusher->sync_threshold || !flusher->flush_queue.size())
-        {
-            // Sync batch is ready. Do it.
-            await_sqe(0);
-            data->iov = { 0 };
-            data->callback = simple_callback_w;
-            my_uring_prep_fsync(sqe, fsync_meta ? bs->meta_fd : bs->data_fd, IORING_FSYNC_DATASYNC);
-            cur_sync->state = 1;
-            wait_count++;
-        resume_1:
-            if (wait_count > 0)
-            {
-                wait_state = 1;
-                return false;
-            }
-            // Sync completed. All previous coroutines waiting for it must be resumed
-            cur_sync->state = 2;
-            bs->ringloop->wakeup();
-        }
-        // Wait until someone else sends and completes a sync.
-    resume_2:
-        if (!cur_sync->state)
-        {
-            wait_state = 2;
-            return false;
-        }
-        cur_sync->ready_count--;
-        if (cur_sync->ready_count == 0)
-        {
-            flusher->syncs.erase(cur_sync);
-        }
-    }
-    return true;
-}
-
-void journal_flusher_co::bitmap_set(void *bitmap, uint64_t start, uint64_t len)
-{
-    if (start == 0)
-    {
-        if (len == 32*bs->bitmap_granularity)
-        {
-            *((uint32_t*)bitmap) = UINT32_MAX;
-            return;
-        }
-        else if (len == 64*bs->bitmap_granularity)
-        {
-            *((uint64_t*)bitmap) = UINT64_MAX;
-            return;
-        }
-    }
-    unsigned bit_start = start / bs->bitmap_granularity;
-    unsigned bit_end = ((start + len) + bs->bitmap_granularity - 1) / bs->bitmap_granularity;
-    while (bit_start < bit_end)
-    {
-        if (!(bit_start & 7) && bit_end >= bit_start+8)
-        {
-            ((uint8_t*)bitmap)[bit_start / 8] = UINT8_MAX;
-            bit_start += 8;
-        }
-        else
-        {
-            ((uint8_t*)bitmap)[bit_start / 8] |= 1 << (bit_start % 8);
-            bit_start++;
-        }
-    }
-}

blockstore_flush.h

@@ -1,101 +0,0 @@
-struct copy_buffer_t
-{
-    uint64_t offset, len;
-    void *buf;
-};
-
-struct meta_sector_t
-{
-    uint64_t offset, len;
-    int state;
-    void *buf;
-    int usage_count;
-};
-
-struct flusher_sync_t
-{
-    bool fsync_meta;
-    int ready_count;
-    int state;
-};
-
-struct flusher_meta_write_t
-{
-    uint64_t sector, pos;
-    bool submitted;
-    void *buf;
-    std::map<uint64_t, meta_sector_t>::iterator it;
-};
-
-class journal_flusher_t;
-
-// Journal flusher coroutine
-class journal_flusher_co
-{
-    blockstore_impl_t *bs;
-    journal_flusher_t *flusher;
-    int wait_state, wait_count;
-    struct io_uring_sqe *sqe;
-    struct ring_data_t *data;
-
-    std::list<flusher_sync_t>::iterator cur_sync;
-
-    obj_ver_id cur;
-    std::map<obj_ver_id, dirty_entry>::iterator dirty_it, dirty_start, dirty_end;
-    std::map<object_id, uint64_t>::iterator repeat_it;
-    std::function<void(ring_data_t*)> simple_callback_r, simple_callback_w;
-
-    bool skip_copy, has_delete, has_empty;
-    spp::sparse_hash_map<object_id, clean_entry>::iterator clean_it;
-    std::vector<copy_buffer_t> v;
-    std::vector<copy_buffer_t>::iterator it;
-    int copy_count;
-    uint64_t clean_loc, old_clean_loc;
-    flusher_meta_write_t meta_old, meta_new;
-    bool clean_init_bitmap;
-    uint64_t clean_bitmap_offset, clean_bitmap_len;
-    void *new_clean_bitmap;
-
-    // local: scan_dirty()
-    uint64_t offset, end_offset, submit_offset, submit_len;
-
-    friend class journal_flusher_t;
-    bool scan_dirty(int wait_base);
-    bool modify_meta_read(uint64_t meta_loc, flusher_meta_write_t &wr, int wait_base);
-    void update_clean_db();
-    bool fsync_batch(bool fsync_meta, int wait_base);
-    void bitmap_set(void *bitmap, uint64_t start, uint64_t len);
-public:
-    journal_flusher_co();
-    bool loop();
-};
-
-// Journal flusher itself
-class journal_flusher_t
-{
-    bool start_forced = false;
-    int flusher_count;
-    int sync_threshold;
-    journal_flusher_co *co;
-    blockstore_impl_t *bs;
-    friend class journal_flusher_co;
-
-    int journal_trim_counter, journal_trim_interval;
-    void* journal_superblock;
-
-    int active_flushers;
-    std::list<flusher_sync_t> syncs;
-    std::map<object_id, uint64_t> sync_to_repeat;
-
-    std::map<uint64_t, meta_sector_t> meta_sectors;
-    std::deque<object_id> flush_queue;
-    std::map<object_id, uint64_t> flush_versions;
-public:
-    journal_flusher_t(int flusher_count, blockstore_impl_t *bs);
-    ~journal_flusher_t();
-    void loop();
-    bool is_active();
-    void force_start();
-    void enqueue_flush(obj_ver_id oid);
-    void unshift_flush(obj_ver_id oid);
-};

blockstore_impl.cpp

@@ -1,458 +0,0 @@
-#include "blockstore_impl.h"
-
-blockstore_impl_t::blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop)
-{
-    assert(sizeof(blockstore_op_private_t) <= BS_OP_PRIVATE_DATA_SIZE);
-    this->ringloop = ringloop;
-    ring_consumer.loop = [this]() { loop(); };
-    ringloop->register_consumer(ring_consumer);
-    initialized = 0;
-    zero_object = (uint8_t*)memalign(MEM_ALIGNMENT, block_size);
-    data_fd = meta_fd = journal.fd = -1;
-    parse_config(config);
-    try
-    {
-        open_data();
-        open_meta();
-        open_journal();
-        calc_lengths();
-        data_alloc = new allocator(block_count);
-    }
-    catch (std::exception & e)
-    {
-        if (data_fd >= 0)
-            close(data_fd);
-        if (meta_fd >= 0 && meta_fd != data_fd)
-            close(meta_fd);
-        if (journal.fd >= 0 && journal.fd != meta_fd)
-            close(journal.fd);
-        throw;
-    }
-    flusher = new journal_flusher_t(flusher_count, this);
-}
-
-blockstore_impl_t::~blockstore_impl_t()
-{
-    delete data_alloc;
-    delete flusher;
-    free(zero_object);
-    ringloop->unregister_consumer(ring_consumer);
-    if (data_fd >= 0)
-        close(data_fd);
-    if (meta_fd >= 0 && meta_fd != data_fd)
-        close(meta_fd);
-    if (journal.fd >= 0 && journal.fd != meta_fd)
-        close(journal.fd);
-    if (metadata_buffer)
-        free(metadata_buffer);
-    if (clean_bitmap)
-        free(clean_bitmap);
-}
-
-bool blockstore_impl_t::is_started()
-{
-    return initialized == 10;
-}
-
-bool blockstore_impl_t::is_stalled()
-{
-    return queue_stall;
-}
-
-// main event loop - produce requests
-void blockstore_impl_t::loop()
-{
-    // FIXME: initialized == 10 is ugly
-    if (initialized != 10)
-    {
-        // read metadata, then journal
-        if (initialized == 0)
-        {
-            metadata_init_reader = new blockstore_init_meta(this);
-            initialized = 1;
-        }
-        if (initialized == 1)
-        {
-            int res = metadata_init_reader->loop();
-            if (!res)
-            {
-                delete metadata_init_reader;
-                metadata_init_reader = NULL;
-                journal_init_reader = new blockstore_init_journal(this);
-                initialized = 2;
-            }
-        }
-        if (initialized == 2)
-        {
-            int res = journal_init_reader->loop();
-            if (!res)
-            {
-                delete journal_init_reader;
-                journal_init_reader = NULL;
-                initialized = 10;
-                ringloop->wakeup();
-            }
-        }
-    }
-    else
-    {
-        // try to submit ops
-        unsigned initial_ring_space = ringloop->space_left();
-        auto cur_sync = in_progress_syncs.begin();
-        while (cur_sync != in_progress_syncs.end())
-        {
-            continue_sync(*cur_sync++);
-        }
-        auto cur = submit_queue.begin();
-        int has_writes = 0;
-        while (cur != submit_queue.end())
-        {
-            auto op_ptr = cur;
-            auto op = *(cur++);
-            // FIXME: This needs some simplification
-            // Writes should not block reads if the ring is not full and reads don't depend on them
-            // In all other cases we should stop submission
-            if (PRIV(op)->wait_for)
-            {
-                check_wait(op);
-#ifdef BLOCKSTORE_DEBUG
-                if (PRIV(op)->wait_for)
-                {
-                    printf("still waiting for %d\n", PRIV(op)->wait_for);
-                }
-#endif
-                if (PRIV(op)->wait_for == WAIT_SQE)
-                {
-                    break;
-                }
-                else if (PRIV(op)->wait_for)
-                {
-                    if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_DELETE)
-                    {
-                        has_writes = 2;
-                    }
-                    continue;
-                }
-            }
-            unsigned ring_space = ringloop->space_left();
-            unsigned prev_sqe_pos = ringloop->save();
-            int dequeue_op = 0;
-            if (op->opcode == BS_OP_READ)
-            {
-                dequeue_op = dequeue_read(op);
-            }
-            else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_DELETE)
-            {
-                if (has_writes == 2)
-                {
-                    // Some writes could not be submitted
-                    break;
-                }
-                dequeue_op = dequeue_write(op);
-                has_writes = dequeue_op ? 1 : 2;
-            }
-            else if (op->opcode == BS_OP_SYNC)
-            {
-                // wait for all small writes to be submitted
-                // wait for all big writes to complete, submit data device fsync
-                // wait for the data device fsync to complete, then submit journal writes for big writes
-                // then submit an fsync operation
-                if (has_writes)
-                {
-                    // Can't submit SYNC before previous writes
-                    continue;
-                }
-                dequeue_op = dequeue_sync(op);
-            }
-            else if (op->opcode == BS_OP_STABLE)
-            {
-                dequeue_op = dequeue_stable(op);
-            }
-            else if (op->opcode == BS_OP_ROLLBACK)
-            {
-                dequeue_op = dequeue_rollback(op);
-            }
-            else if (op->opcode == BS_OP_LIST)
-            {
-                process_list(op);
-                dequeue_op = true;
-            }
-            if (dequeue_op)
-            {
-                submit_queue.erase(op_ptr);
-            }
-            else
-            {
-                ringloop->restore(prev_sqe_pos);
-                if (PRIV(op)->wait_for == WAIT_SQE)
-                {
-                    PRIV(op)->wait_detail = 1 + ring_space;
-                    // ring is full, stop submission
-                    break;
-                }
-            }
-        }
-        if (!readonly)
-        {
-            flusher->loop();
-        }
-        int ret = ringloop->submit();
-        if (ret < 0)
-        {
-            throw std::runtime_error(std::string("io_uring_submit: ") + strerror(-ret));
-        }
-        if ((initial_ring_space - ringloop->space_left()) > 0)
-        {
-            live = true;
-        }
-        queue_stall = !live && !ringloop->get_loop_again();
-        live = false;
-    }
-}
-
-bool blockstore_impl_t::is_safe_to_stop()
-{
-    // It's safe to stop blockstore when there are no in-flight operations,
-    // no in-progress syncs and flusher isn't doing anything
-    if (submit_queue.size() > 0 || in_progress_syncs.size() > 0 || !readonly && flusher->is_active())
-    {
-        return false;
-    }
-    if (unsynced_big_writes.size() > 0 || unsynced_small_writes.size() > 0)
-    {
-        if (!readonly && !stop_sync_submitted)
-        {
-            // We should sync the blockstore before unmounting
-            blockstore_op_t *op = new blockstore_op_t;
-            op->opcode = BS_OP_SYNC;
-            op->buf = NULL;
-            op->callback = [](blockstore_op_t *op)
-            {
-                delete op;
-            };
-            enqueue_op(op);
-            stop_sync_submitted = true;
-        }
-        return false;
-    }
-    return true;
-}
-
-void blockstore_impl_t::check_wait(blockstore_op_t *op)
-{
-    if (PRIV(op)->wait_for == WAIT_SQE)
-    {
-        if (ringloop->space_left() < PRIV(op)->wait_detail)
-        {
-            // stop submission if there's still no free space
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_IN_FLIGHT)
-    {
-        auto dirty_it = dirty_db.find((obj_ver_id){
-            .oid = op->oid,
-            .version = PRIV(op)->wait_detail,
-        });
-        if (dirty_it != dirty_db.end() && IS_IN_FLIGHT(dirty_it->second.state))
-        {
-            // do not submit
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_JOURNAL)
-    {
-        if (journal.used_start == PRIV(op)->wait_detail)
-        {
-            // do not submit
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_JOURNAL_BUFFER)
-    {
-        int next = ((journal.cur_sector + 1) % journal.sector_count);
-        if (journal.sector_info[next].usage_count > 0 ||
-            journal.sector_info[next].dirty)
-        {
-            // do not submit
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else if (PRIV(op)->wait_for == WAIT_FREE)
-    {
-        if (!data_alloc->get_free_count() && !flusher->is_active())
-        {
-            return;
-        }
-        PRIV(op)->wait_for = 0;
-    }
-    else
-    {
-        throw std::runtime_error("BUG: op->wait_for value is unexpected");
-    }
-}
-
-void blockstore_impl_t::enqueue_op(blockstore_op_t *op, bool first)
-{
-    if (op->opcode < BS_OP_MIN || op->opcode > BS_OP_MAX ||
-        ((op->opcode == BS_OP_READ || op->opcode == BS_OP_WRITE) && (
-            op->offset >= block_size ||
-            op->len > block_size-op->offset ||
-            (op->len % disk_alignment)
-        )) ||
-        readonly && op->opcode != BS_OP_READ ||
-        first && op->opcode == BS_OP_WRITE)
-    {
-        // Basic verification not passed
-        op->retval = -EINVAL;
-        op->callback(op);
-        return;
-    }
-    if (op->opcode == BS_OP_SYNC_STAB_ALL)
-    {
-        std::function<void(blockstore_op_t*)> *old_callback = new std::function<void(blockstore_op_t*)>(op->callback);
-        op->opcode = BS_OP_SYNC;
-        op->callback = [this, old_callback](blockstore_op_t *op)
-        {
-            if (op->retval >= 0 && unstable_writes.size() > 0)
-            {
-                op->opcode = BS_OP_STABLE;
-                op->len = unstable_writes.size();
-                obj_ver_id *vers = new obj_ver_id[op->len];
-                op->buf = vers;
-                int i = 0;
-                for (auto it = unstable_writes.begin(); it != unstable_writes.end(); it++, i++)
-                {
-                    vers[i] = {
-                        .oid = it->first,
-                        .version = it->second,
-                    };
-                }
-                unstable_writes.clear();
-                op->callback = [this, old_callback](blockstore_op_t *op)
-                {
-                    obj_ver_id *vers = (obj_ver_id*)op->buf;
-                    delete[] vers;
-                    op->buf = NULL;
-                    (*old_callback)(op);
-                    delete old_callback;
-                };
-                this->enqueue_op(op);
-            }
-            else
-            {
-                (*old_callback)(op);
-                delete old_callback;
-            }
-        };
-    }
-    if (op->opcode == BS_OP_WRITE && !enqueue_write(op))
-    {
-        op->callback(op);
-        return;
-    }
-    if (0 && op->opcode == BS_OP_SYNC && immediate_commit)
-    {
-        op->retval = 0;
-        op->callback(op);
-        return;
-    }
-    // Call constructor without allocating memory. We'll call destructor before returning op back
-    new ((void*)op->private_data) blockstore_op_private_t;
-    PRIV(op)->wait_for = 0;
-    PRIV(op)->sync_state = 0;
-    PRIV(op)->pending_ops = 0;
-    if (!first)
-    {
-        submit_queue.push_back(op);
-    }
-    else
-    {
-        submit_queue.push_front(op);
-    }
-    ringloop->wakeup();
-}
-
-void blockstore_impl_t::process_list(blockstore_op_t *op)
-{
-    // Count objects
-    uint32_t list_pg = op->offset;
-    uint32_t pg_count = op->len;
-    uint64_t parity_block_size = op->oid.stripe;
-    if (pg_count != 0 && (parity_block_size < MIN_BLOCK_SIZE || list_pg >= pg_count))
-    {
-        op->retval = -EINVAL;
-        FINISH_OP(op);
-        return;
-    }
-    uint64_t stable_count = 0;
-    if (pg_count > 0)
-    {
-        for (auto it = clean_db.begin(); it != clean_db.end(); it++)
-        {
-            uint32_t pg = (it->first.inode + it->first.stripe / parity_block_size) % pg_count;
-            if (pg == list_pg)
-            {
-                stable_count++;
-            }
-        }
-    }
-    else
-    {
-        stable_count = clean_db.size();
-    }
-    uint64_t total_count = stable_count;
-    for (auto it = dirty_db.begin(); it != dirty_db.end(); it++)
-    {
-        if (!pg_count || ((it->first.oid.inode + it->first.oid.stripe / parity_block_size) % pg_count) == list_pg)
-        {
-            if (IS_STABLE(it->second.state))
-            {
-                stable_count++;
-            }
-            total_count++;
-        }
-    }
-    // Allocate memory
-    op->version = stable_count;
-    op->retval = total_count;
-    op->buf = malloc(sizeof(obj_ver_id) * total_count);
-    if (!op->buf)
-    {
-        op->retval = -ENOMEM;
-        FINISH_OP(op);
-        return;
-    }
-    obj_ver_id *vers = (obj_ver_id*)op->buf;
-    int i = 0;
-    for (auto it = clean_db.begin(); it != clean_db.end(); it++)
-    {
-        if (!pg_count || ((it->first.inode + it->first.stripe / parity_block_size) % pg_count) == list_pg)
-        {
-            vers[i++] = {
-                .oid = it->first,
-                .version = it->second.version,
-            };
-        }
-    }
-    int j = stable_count;
-    for (auto it = dirty_db.begin(); it != dirty_db.end(); it++)
-    {
-        if (!pg_count || ((it->first.oid.inode + it->first.oid.stripe / parity_block_size) % pg_count) == list_pg)
-        {
-            if (IS_STABLE(it->second.state))
-            {
-                vers[i++] = it->first;
-            }
-            else
-            {
-                vers[j++] = it->first;
-            }
-        }
-    }
-    FINISH_OP(op);
-}

blockstore_impl.h

@@ -1,320 +0,0 @@
-#pragma once
-
-#include "blockstore.h"
-#include "timerfd_interval.h"
-
-#include <sys/types.h>
-#include <sys/ioctl.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <malloc.h>
-#include <linux/fs.h>
-
-#include <vector>
-#include <list>
-#include <deque>
-#include <new>
-
-#include "sparsepp/sparsepp/spp.h"
-
-#include "allocator.h"
-
-//#define BLOCKSTORE_DEBUG
-
-// States are not stored on disk. Instead, they're deduced from the journal
-// FIXME: Rename to BS_ST_*
-
-#define ST_J_IN_FLIGHT 1
-#define ST_J_SUBMITTED 2
-#define ST_J_WRITTEN 3
-#define ST_J_SYNCED 4
-#define ST_J_STABLE 5
-
-#define ST_D_IN_FLIGHT 15
-#define ST_D_SUBMITTED 16
-#define ST_D_WRITTEN 17
-#define ST_D_META_WRITTEN 19
-#define ST_D_META_SYNCED 20
-#define ST_D_STABLE 21
-
-#define ST_DEL_IN_FLIGHT 31
-#define ST_DEL_SUBMITTED 32
-#define ST_DEL_WRITTEN 33
-#define ST_DEL_SYNCED 34
-#define ST_DEL_STABLE 35
-
-#define ST_CURRENT 48
-
-#define IS_IN_FLIGHT(st) (st == ST_J_IN_FLIGHT || st == ST_D_IN_FLIGHT || st == ST_DEL_IN_FLIGHT || st == ST_J_SUBMITTED || st == ST_D_SUBMITTED || st == ST_DEL_SUBMITTED)
-#define IS_STABLE(st) (st == ST_J_STABLE || st == ST_D_STABLE || st == ST_DEL_STABLE || st == ST_CURRENT)
-#define IS_SYNCED(st) (IS_STABLE(st) || st == ST_J_SYNCED || st == ST_D_META_SYNCED || st == ST_DEL_SYNCED)
-#define IS_JOURNAL(st) (st >= ST_J_SUBMITTED && st <= ST_J_STABLE)
-#define IS_BIG_WRITE(st) (st >= ST_D_SUBMITTED && st <= ST_D_STABLE)
-#define IS_DELETE(st) (st >= ST_DEL_SUBMITTED && st <= ST_DEL_STABLE)
-#define IS_UNSYNCED(st) (st >= ST_J_SUBMITTED && st <= ST_J_WRITTEN || st >= ST_D_SUBMITTED && st <= ST_D_META_WRITTEN || st >= ST_DEL_SUBMITTED && st <= ST_DEL_WRITTEN)
-
-#define BS_SUBMIT_GET_SQE(sqe, data) \
-    BS_SUBMIT_GET_ONLY_SQE(sqe); \
-    struct ring_data_t *data = ((ring_data_t*)sqe->user_data)
-
-#define BS_SUBMIT_GET_ONLY_SQE(sqe) \
-    struct io_uring_sqe *sqe = get_sqe();\
-    if (!sqe)\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_for = WAIT_SQE;\
-        return 0;\
-    }
-
-#define BS_SUBMIT_GET_SQE_DECL(sqe) \
-    sqe = get_sqe();\
-    if (!sqe)\
-    {\
-        /* Pause until there are more requests available */\
-        PRIV(op)->wait_for = WAIT_SQE;\
-        return 0;\
-    }
-
-#include "blockstore_journal.h"
-
-// 24 bytes + block bitmap per "clean" entry on disk with fixed metadata tables
-// FIXME: maybe add crc32's to metadata
-struct __attribute__((__packed__)) clean_disk_entry
-{
-    object_id oid;
-    uint64_t version;
-    uint8_t bitmap[];
-};
-
-// 32 = 16 + 16 bytes per "clean" entry in memory (object_id => clean_entry)
-struct __attribute__((__packed__)) clean_entry
-{
-    uint64_t version;
-    uint64_t location;
-};
-
-// 56 = 24 + 32 bytes per dirty entry in memory (obj_ver_id => dirty_entry)
-struct __attribute__((__packed__)) dirty_entry
-{
-    uint32_t state;
-    uint32_t flags;    // unneeded, but present for alignment
-    uint64_t location; // location in either journal or data -> in BYTES
-    uint32_t offset;   // data offset within object (stripe)
-    uint32_t len;      // data length
-    uint64_t journal_sector; // journal sector used for this entry
-};
-
-// - Sync must be submitted after previous writes/deletes (not before!)
-// - Reads to the same object must be submitted after previous writes/deletes
-//   are written (not necessarily synced) in their location. This is because we
-//   rely on read-modify-write for erasure coding and we must return new data
-//   to calculate parity for subsequent writes
-// - Writes may be submitted in any order, because they don't overlap. Each write
-//   goes into a new location - either on the journal device or on the data device
-// - Stable (stabilize) must be submitted after sync of that object is completed
-//   It's even OK to return an error to the caller if that object is not synced yet
-// - Journal trim may be processed only after all versions are moved to
-//   the main storage AND after all read operations for older versions complete
-// - If an operation can not be submitted because the ring is full
-//   we should stop submission of other operations. Otherwise some "scatter" reads
-//   may end up blocked for a long time.
-// Otherwise, the submit order is free, that is all operations may be submitted immediately
-// In fact, adding a write operation must immediately result in dirty_db being populated
-
-// Suspend operation until there are more free SQEs
-#define WAIT_SQE 1
-// Suspend operation until version <wait_detail> of object <oid> is written
-#define WAIT_IN_FLIGHT 2
-// Suspend operation until there are <wait_detail> bytes of free space in the journal on disk
-#define WAIT_JOURNAL 3
-// Suspend operation until the next journal sector buffer is free
-#define WAIT_JOURNAL_BUFFER 4
-// Suspend operation until there is some free space on the data device
-#define WAIT_FREE 5
-
-struct fulfill_read_t
-{
-    uint64_t offset, len;
-};
-
-#define PRIV(op) ((blockstore_op_private_t*)(op)->private_data)
-#define FINISH_OP(op) PRIV(op)->~blockstore_op_private_t(); op->callback(op)
-
-struct blockstore_op_private_t
-{
-    // Wait status
-    int wait_for;
-    uint64_t wait_detail;
-    int pending_ops;
-
-    // Read
-    std::vector<fulfill_read_t> read_vec;
-
-    // Sync, write
-    uint64_t min_used_journal_sector, max_used_journal_sector;
-
-    // Write
-    struct iovec iov_zerofill[3];
-
-    // Sync
-    std::vector<obj_ver_id> sync_big_writes, sync_small_writes;
-    int sync_small_checked, sync_big_checked;
-    std::list<blockstore_op_t*>::iterator in_progress_ptr;
-    int sync_state, prev_sync_count;
-};
-
-typedef std::map<obj_ver_id, dirty_entry> blockstore_dirty_db_t;
-
-#include "blockstore_init.h"
-
-#include "blockstore_flush.h"
-
-class blockstore_impl_t
-{
-    /******* OPTIONS *******/
-    std::string data_device, meta_device, journal_device;
-    uint32_t block_size;
-    uint64_t meta_offset;
-    uint64_t data_offset;
-    uint64_t cfg_journal_size;
-    // Required write alignment and journal/metadata/data areas' location alignment
-    uint32_t disk_alignment = 512;
-    // Journal block size - minimum_io_size of the journal device is the best choice
-    uint64_t journal_block_size = 512;
-    // Metadata block size - minimum_io_size of the metadata device is the best choice
-    uint64_t meta_block_size = 512;
-    // Sparse write tracking granularity. 4 KB is a good choice. Must be a multiple of disk_alignment
-    uint64_t bitmap_granularity = 4096;
-    bool readonly = false;
-    // It is safe to disable fsync() if drive write cache is writethrough
-    bool disable_data_fsync = false, disable_meta_fsync = false, disable_journal_fsync = false;
-    // Enable if you want every operation to be executed with an "implicit fsync"
-    // FIXME Not implemented yet
-    bool immediate_commit = false;
-    bool inmemory_meta = false;
-    int flusher_count;
-    /******* END OF OPTIONS *******/
-
-    struct ring_consumer_t ring_consumer;
-
-    // Another option is https://github.com/algorithm-ninja/cpp-btree
-    spp::sparse_hash_map<object_id, clean_entry> clean_db;
-    uint8_t *clean_bitmap = NULL;
-    blockstore_dirty_db_t dirty_db;
-    std::list<blockstore_op_t*> submit_queue; // FIXME: funny thing is that vector is better here
-    std::vector<obj_ver_id> unsynced_big_writes, unsynced_small_writes;
-    std::list<blockstore_op_t*> in_progress_syncs; // ...and probably here, too
-    allocator *data_alloc = NULL;
-    uint8_t *zero_object;
-
-    uint32_t block_order;
-    uint64_t block_count;
-    uint32_t clean_entry_bitmap_size = 0, clean_entry_size = 0;
-
-    int meta_fd;
-    int data_fd;
-    uint64_t meta_size, meta_area, meta_len;
-    uint64_t data_size, data_len;
-
-    void *metadata_buffer = NULL;
-
-    struct journal_t journal;
-    journal_flusher_t *flusher;
-
-    bool live = false, queue_stall = false;
-    ring_loop_t *ringloop;
-
-    bool stop_sync_submitted;
-
-    inline struct io_uring_sqe* get_sqe()
-    {
-        return ringloop->get_sqe();
-    }
-
-    friend class blockstore_init_meta;
-    friend class blockstore_init_journal;
-    friend class blockstore_journal_check_t;
-    friend class journal_flusher_t;
-    friend class journal_flusher_co;
-
-    void parse_config(blockstore_config_t & config);
-    void calc_lengths();
-    void open_data();
-    void open_meta();
-    void open_journal();
-
-    // Asynchronous init
-    int initialized;
-    int metadata_buf_size;
-    blockstore_init_meta* metadata_init_reader;
-    blockstore_init_journal* journal_init_reader;
-
-    void check_wait(blockstore_op_t *op);
-
-    // Read
-    int dequeue_read(blockstore_op_t *read_op);
-    int fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-        uint32_t item_state, uint64_t item_version, uint64_t item_location);
-    int fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
-        uint32_t item_state, uint64_t item_version);
-    void handle_read_event(ring_data_t *data, blockstore_op_t *op);
-
-    // Write
-    bool enqueue_write(blockstore_op_t *op);
-    int dequeue_write(blockstore_op_t *op);
-    int dequeue_del(blockstore_op_t *op);
-    void ack_write(blockstore_op_t *op);
-    void release_journal_sectors(blockstore_op_t *op);
-    void handle_write_event(ring_data_t *data, blockstore_op_t *op);
-
-    // Sync
-    int dequeue_sync(blockstore_op_t *op);
-    void handle_sync_event(ring_data_t *data, blockstore_op_t *op);
-    int continue_sync(blockstore_op_t *op);
-    void ack_one_sync(blockstore_op_t *op);
-    int ack_sync(blockstore_op_t *op);
-
-    // Stabilize
-    int dequeue_stable(blockstore_op_t *op);
-    void handle_stable_event(ring_data_t *data, blockstore_op_t *op);
-    void stabilize_object(object_id oid, uint64_t max_ver);
-
-    // Rollback
-    int dequeue_rollback(blockstore_op_t *op);
-    void handle_rollback_event(ring_data_t *data, blockstore_op_t *op);
-    void erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc);
-
-    // List
-    void process_list(blockstore_op_t *op);
-
-public:
-
-    blockstore_impl_t(blockstore_config_t & config, ring_loop_t *ringloop);
-    ~blockstore_impl_t();
-
-    // Event loop
-    void loop();
-
-    // Returns true when blockstore is ready to process operations
-    // (Although you're free to enqueue them before that)
-    bool is_started();
-
-    // Returns true when it's safe to destroy the instance. If destroying the instance
-    // requires to purge some queues, starts that process. Should be called in the event
-    // loop until it returns true.
-    bool is_safe_to_stop();
-
-    // Returns true if stalled
-    bool is_stalled();
-
-    // Submission
-    void enqueue_op(blockstore_op_t *op, bool first = false);
-
-    // Unstable writes are added here (map of object_id -> version)
-    std::unordered_map<object_id, uint64_t> unstable_writes;
-
-    inline uint32_t get_block_size() { return block_size; }
-    inline uint64_t get_block_count() { return block_count; }
-    inline uint32_t get_disk_alignment() { return disk_alignment; }
-};

blockstore_init.cpp

@@ -1,691 +0,0 @@
-#include "blockstore_impl.h"
-
-blockstore_init_meta::blockstore_init_meta(blockstore_impl_t *bs)
-{
-    this->bs = bs;
-}
-
-void blockstore_init_meta::handle_event(ring_data_t *data)
-{
-    if (data->res <= 0)
-    {
-        throw std::runtime_error(
-            std::string("read metadata failed at offset ") + std::to_string(metadata_read) +
-            std::string(": ") + strerror(-data->res)
-        );
-    }
-    prev_done = data->res > 0 ? submitted : 0;
-    done_len = data->res;
-    done_pos = metadata_read;
-    metadata_read += data->res;
-    submitted = 0;
-}
-
-int blockstore_init_meta::loop()
-{
-    if (wait_state == 1)
-        goto resume_1;
-    printf("Reading blockstore metadata\n");
-    if (bs->inmemory_meta)
-        metadata_buffer = bs->metadata_buffer;
-    else
-        metadata_buffer = memalign(MEM_ALIGNMENT, 2*bs->metadata_buf_size);
-    if (!metadata_buffer)
-        throw std::runtime_error("Failed to allocate metadata read buffer");
-    while (1)
-    {
-    resume_1:
-        if (submitted)
-        {
-            wait_state = 1;
-            return 1;
-        }
-        if (metadata_read < bs->meta_len)
-        {
-            sqe = bs->get_sqe();
-            if (!sqe)
-            {
-                throw std::runtime_error("io_uring is full while trying to read metadata");
-            }
-            data = ((ring_data_t*)sqe->user_data);
-            data->iov = {
-                metadata_buffer + (bs->inmemory_meta
-                    ? metadata_read
-                    : (prev == 1 ? bs->metadata_buf_size : 0)),
-                bs->meta_len - metadata_read > bs->metadata_buf_size ? bs->metadata_buf_size : bs->meta_len - metadata_read,
-            };
-            data->callback = [this](ring_data_t *data) { handle_event(data); };
-            my_uring_prep_readv(sqe, bs->meta_fd, &data->iov, 1, bs->meta_offset + metadata_read);
-            bs->ringloop->submit();
-            submitted = (prev == 1 ? 2 : 1);
-            prev = submitted;
-        }
-        if (prev_done)
-        {
-            void *done_buf = bs->inmemory_meta
-                ? (metadata_buffer + done_pos)
-                : (metadata_buffer + (prev_done == 2 ? bs->metadata_buf_size : 0));
-            unsigned count = bs->meta_block_size / bs->clean_entry_size;
-            for (int sector = 0; sector < done_len; sector += bs->meta_block_size)
-            {
-                // handle <count> entries
-                handle_entries(done_buf + sector, count, bs->block_order);
-                done_cnt += count;
-            }
-            prev_done = 0;
-            done_len = 0;
-        }
-        if (!submitted)
-        {
-            break;
-        }
-    }
-    // metadata read finished
-    printf("Metadata entries loaded: %lu, free blocks: %lu / %lu\n", entries_loaded, bs->data_alloc->get_free_count(), bs->block_count);
-    if (!bs->inmemory_meta)
-    {
-        free(metadata_buffer);
-        metadata_buffer = NULL;
-    }
-    return 0;
-}
-
-void blockstore_init_meta::handle_entries(void* entries, unsigned count, int block_order)
-{
-    for (unsigned i = 0; i < count; i++)
-    {
-        clean_disk_entry *entry = (clean_disk_entry*)(entries + i*bs->clean_entry_size);
-        if (!bs->inmemory_meta && bs->clean_entry_bitmap_size)
-        {
-            memcpy(bs->clean_bitmap + (done_cnt+i)*bs->clean_entry_bitmap_size, &entry->bitmap, bs->clean_entry_bitmap_size);
-        }
-        if (entry->oid.inode > 0)
-        {
-            auto clean_it = bs->clean_db.find(entry->oid);
-            if (clean_it == bs->clean_db.end() || clean_it->second.version < entry->version)
-            {
-                if (clean_it != bs->clean_db.end())
-                {
-                    // free the previous block
-#ifdef BLOCKSTORE_DEBUG
-                    printf("Free block %lu\n", clean_it->second.location >> bs->block_order);
-#endif
-                    bs->data_alloc->set(clean_it->second.location >> block_order, false);
-                }
-                entries_loaded++;
-#ifdef BLOCKSTORE_DEBUG
-                printf("Allocate block (clean entry) %lu: %lu:%lu v%lu\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version);
-#endif
-                bs->data_alloc->set(done_cnt+i, true);
-                bs->clean_db[entry->oid] = (struct clean_entry){
-                    .version = entry->version,
-                    .location = (done_cnt+i) << block_order,
-                };
-            }
-            else
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("Old clean entry %lu: %lu:%lu v%lu\n", done_cnt+i, entry->oid.inode, entry->oid.stripe, entry->version);
-#endif
-            }
-        }
-    }
-}
-
-blockstore_init_journal::blockstore_init_journal(blockstore_impl_t *bs)
-{
-    this->bs = bs;
-    next_free = bs->journal.block_size;
-    simple_callback = [this](ring_data_t *data1)
-    {
-        if (data1->res != data1->iov.iov_len)
-        {
-            throw std::runtime_error(std::string("I/O operation failed while reading journal: ") + strerror(-data1->res));
-        }
-        wait_count--;
-    };
-}
-
-bool iszero(uint64_t *buf, int len)
-{
-    for (int i = 0; i < len; i++)
-        if (buf[i] != 0)
-            return false;
-    return true;
-}
-
-void blockstore_init_journal::handle_event(ring_data_t *data1)
-{
-    if (data1->res <= 0)
-    {
-        throw std::runtime_error(
-            std::string("read journal failed at offset ") + std::to_string(journal_pos) +
-            std::string(": ") + strerror(-data1->res)
-        );
-    }
-    done.push_back({
-        .buf = submitted_buf,
-        .pos = journal_pos,
-        .len = (uint64_t)data1->res,
-    });
-    journal_pos += data1->res;
-    if (journal_pos >= bs->journal.len)
-    {
-        // Continue from the beginning
-        journal_pos = bs->journal.block_size;
-        wrapped = true;
-    }
-    submitted_buf = NULL;
-}
-
-#define GET_SQE() \
-    sqe = bs->get_sqe();\
-    if (!sqe)\
-        throw std::runtime_error("io_uring is full while trying to read journal");\
-    data = ((ring_data_t*)sqe->user_data)
-
-int blockstore_init_journal::loop()
-{
-    if (wait_state == 1)
-        goto resume_1;
-    else if (wait_state == 2)
-        goto resume_2;
-    else if (wait_state == 3)
-        goto resume_3;
-    else if (wait_state == 4)
-        goto resume_4;
-    else if (wait_state == 5)
-        goto resume_5;
-    else if (wait_state == 6)
-        goto resume_6;
-    else if (wait_state == 7)
-        goto resume_7;
-    printf("Reading blockstore journal\n");
-    if (!bs->journal.inmemory)
-    {
-        submitted_buf = memalign(MEM_ALIGNMENT, 2*bs->journal.block_size);
-        if (!submitted_buf)
-            throw std::bad_alloc();
-    }
-    else
-        submitted_buf = bs->journal.buffer;
-    // Read first block of the journal
-    sqe = bs->get_sqe();
-    if (!sqe)
-        throw std::runtime_error("io_uring is full while trying to read journal");
-    data = ((ring_data_t*)sqe->user_data);
-    data->iov = { submitted_buf, bs->journal.block_size };
-    data->callback = simple_callback;
-    my_uring_prep_readv(sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset);
-    bs->ringloop->submit();
-    wait_count = 1;
-resume_1:
-    if (wait_count > 0)
-    {
-        wait_state = 1;
-        return 1;
-    }
-    if (iszero((uint64_t*)submitted_buf, 3))
-    {
-        // Journal is empty
-        // FIXME handle this wrapping to journal_block_size better (maybe)
-        bs->journal.used_start = bs->journal.block_size;
-        bs->journal.next_free = bs->journal.block_size;
-        // Initialize journal "superblock" and the first block
-        memset(submitted_buf, 0, 2*bs->journal.block_size);
-        *((journal_entry_start*)submitted_buf) = {
-            .crc32 = 0,
-            .magic = JOURNAL_MAGIC,
-            .type = JE_START,
-            .size = sizeof(journal_entry_start),
-            .reserved = 0,
-            .journal_start = bs->journal.block_size,
-        };
-        ((journal_entry_start*)submitted_buf)->crc32 = je_crc32((journal_entry*)submitted_buf);
-        if (bs->readonly)
-        {
-            printf("Skipping journal initialization because blockstore is readonly\n");
-        }
-        else
-        {
-            // Cool effect. Same operations result in journal replay.
-            // FIXME: Randomize initial crc32. Track crc32 when trimming.
-            printf("Resetting journal\n");
-            GET_SQE();
-            data->iov = (struct iovec){ submitted_buf, 2*bs->journal.block_size };
-            data->callback = simple_callback;
-            my_uring_prep_writev(sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset);
-            wait_count++;
-            bs->ringloop->submit();
-        resume_6:
-            if (wait_count > 0)
-            {
-                wait_state = 6;
-                return 1;
-            }
-            if (!bs->disable_journal_fsync)
-            {
-                GET_SQE();
-                my_uring_prep_fsync(sqe, bs->journal.fd, IORING_FSYNC_DATASYNC);
-                data->iov = { 0 };
-                data->callback = simple_callback;
-                wait_count++;
-                bs->ringloop->submit();
-            }
-        resume_4:
-            if (wait_count > 0)
-            {
-                wait_state = 4;
-                return 1;
-            }
-        }
-        if (!bs->journal.inmemory)
-        {
-            free(submitted_buf);
-        }
-    }
-    else
-    {
-        // First block always contains a single JE_START entry
-        je_start = (journal_entry_start*)submitted_buf;
-        if (je_start->magic != JOURNAL_MAGIC ||
-            je_start->type != JE_START ||
-            je_start->size != sizeof(journal_entry_start) ||
-            je_crc32((journal_entry*)je_start) != je_start->crc32)
-        {
-            // Entry is corrupt
-            throw std::runtime_error("first entry of the journal is corrupt");
-        }
-        next_free = journal_pos = bs->journal.used_start = je_start->journal_start;
-        if (!bs->journal.inmemory)
-            free(submitted_buf);
-        submitted_buf = NULL;
-        crc32_last = 0;
-        // Read journal
-        while (1)
-        {
-        resume_2:
-            if (submitted_buf)
-            {
-                wait_state = 2;
-                return 1;
-            }
-            if (!wrapped || journal_pos < bs->journal.used_start)
-            {
-                GET_SQE();
-                uint64_t end = bs->journal.len;
-                if (journal_pos < bs->journal.used_start)
-                    end = bs->journal.used_start;
-                if (!bs->journal.inmemory)
-                    submitted_buf = memalign(MEM_ALIGNMENT, JOURNAL_BUFFER_SIZE);
-                else
-                    submitted_buf = bs->journal.buffer + journal_pos;
-                data->iov = {
-                    submitted_buf,
-                    end - journal_pos < JOURNAL_BUFFER_SIZE ? end - journal_pos : JOURNAL_BUFFER_SIZE,
-                };
-                data->callback = [this](ring_data_t *data1) { handle_event(data1); };
-                my_uring_prep_readv(sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset + journal_pos);
-                bs->ringloop->submit();
-            }
-            while (done.size() > 0)
-            {
-                handle_res = handle_journal_part(done[0].buf, done[0].pos, done[0].len);
-                if (handle_res == 0)
-                {
-                    // journal ended
-                    // zero out corrupted entry, if required
-                    if (init_write_buf && !bs->readonly)
-                    {
-                        GET_SQE();
-                        data->iov = { init_write_buf, bs->journal.block_size };
-                        data->callback = simple_callback;
-                        my_uring_prep_writev(sqe, bs->journal.fd, &data->iov, 1, bs->journal.offset + init_write_sector);
-                        wait_count++;
-                        bs->ringloop->submit();
-                    resume_7:
-                        if (wait_count > 0)
-                        {
-                            wait_state = 7;
-                            return 1;
-                        }
-                        if (!bs->disable_journal_fsync)
-                        {
-                            GET_SQE();
-                            data->iov = { 0 };
-                            data->callback = simple_callback;
-                            my_uring_prep_fsync(sqe, bs->journal.fd, IORING_FSYNC_DATASYNC);
-                            wait_count++;
-                            bs->ringloop->submit();
-                        }
-                    resume_5:
-                        if (wait_count > 0)
-                        {
-                            wait_state = 5;
-                            return 1;
-                        }
-                    }
-                    // wait for the next read to complete, then stop
-                resume_3:
-                    if (submitted_buf)
-                    {
-                        wait_state = 3;
-                        return 1;
-                    }
-                    // free buffers
-                    if (!bs->journal.inmemory)
-                        for (auto & e: done)
-                            free(e.buf);
-                    done.clear();
-                    break;
-                }
-                else if (handle_res == 1)
-                {
-                    // OK, remove it
-                    if (!bs->journal.inmemory)
-                    {
-                        free(done[0].buf);
-                    }
-                    done.erase(done.begin());
-                }
-                else if (handle_res == 2)
-                {
-                    // Need to wait for more reads
-                    break;
-                }
-            }
-            if (!submitted_buf)
-            {
-                break;
-            }
-        }
-    }
-    // Trim journal on start so we don't stall when all entries are older
-    bs->journal.trim();
-    printf(
-        "Journal entries loaded: %lu, free journal space: %lu bytes (%lu..%lu is used), free blocks: %lu / %lu\n",
-        entries_loaded,
-        (bs->journal.next_free >= bs->journal.used_start
-            ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start)
-            : bs->journal.used_start - bs->journal.next_free),
-        bs->journal.used_start, bs->journal.next_free,
-        bs->data_alloc->get_free_count(), bs->block_count
-    );
-    bs->journal.crc32_last = crc32_last;
-    return 0;
-}
-
-int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, uint64_t len)
-{
-    uint64_t proc_pos, pos;
-    if (continue_pos != 0)
-    {
-        proc_pos = (continue_pos / bs->journal.block_size) * bs->journal.block_size;
-        pos = continue_pos % bs->journal.block_size;
-        continue_pos = 0;
-        goto resume;
-    }
-    while (next_free >= done_pos && next_free < done_pos+len)
-    {
-        proc_pos = next_free;
-        pos = 0;
-        next_free += bs->journal.block_size;
-        if (next_free >= bs->journal.len)
-        {
-            next_free = bs->journal.block_size;
-        }
-    resume:
-        while (pos < bs->journal.block_size)
-        {
-            journal_entry *je = (journal_entry*)(buf + proc_pos - done_pos + pos);
-            if (je->magic != JOURNAL_MAGIC || je_crc32(je) != je->crc32 ||
-                je->type < JE_SMALL_WRITE || je->type > JE_DELETE || started && je->crc32_prev != crc32_last)
-            {
-                if (pos == 0)
-                {
-                    // invalid entry in the beginning, this is definitely the end of the journal
-                    bs->journal.next_free = proc_pos;
-                    return 0;
-                }
-                else
-                {
-                    // allow partially filled sectors
-                    break;
-                }
-            }
-            if (je->type == JE_SMALL_WRITE)
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("je_small_write oid=%lu:%lu ver=%lu offset=%u len=%u\n", je->small_write.oid.inode, je->small_write.oid.stripe, je->small_write.version, je->small_write.offset, je->small_write.len);
-#endif
-                // oid, version, offset, len
-                uint64_t prev_free = next_free;
-                if (next_free + je->small_write.len > bs->journal.len)
-                {
-                    // data continues from the beginning of the journal
-                    next_free = bs->journal.block_size;
-                }
-                uint64_t location = next_free;
-                next_free += je->small_write.len;
-                if (next_free >= bs->journal.len)
-                {
-                    next_free = bs->journal.block_size;
-                }
-                if (location != je->small_write.data_offset)
-                {
-                    char err[1024];
-                    snprintf(err, 1024, "BUG: calculated journal data offset (%lu) != stored journal data offset (%lu)", location, je->small_write.data_offset);
-                    throw std::runtime_error(err);
-                }
-                uint32_t data_crc32 = 0;
-                if (location >= done_pos && location+je->small_write.len <= done_pos+len)
-                {
-                    // data is within this buffer
-                    data_crc32 = crc32c(0, buf + location - done_pos, je->small_write.len);
-                }
-                else
-                {
-                    // this case is even more interesting because we must carry data crc32 check to next buffer(s)
-                    uint64_t covered = 0;
-                    for (int i = 0; i < done.size(); i++)
-                    {
-                        if (location+je->small_write.len > done[i].pos &&
-                            location < done[i].pos+done[i].len)
-                        {
-                            uint64_t part_end = (location+je->small_write.len < done[i].pos+done[i].len
-                                ? location+je->small_write.len : done[i].pos+done[i].len);
-                            uint64_t part_begin = (location < done[i].pos ? done[i].pos : location);
-                            covered += part_end - part_begin;
-                            data_crc32 = crc32c(data_crc32, done[i].buf + part_begin - done[i].pos, part_end - part_begin);
-                        }
-                    }
-                    if (covered < je->small_write.len)
-                    {
-                        continue_pos = proc_pos+pos;
-                        next_free = prev_free;
-                        return 2;
-                    }
-                }
-                if (data_crc32 != je->small_write.crc32_data)
-                {
-                    // journal entry is corrupt, stop here
-                    // interesting thing is that we must clear the corrupt entry if we're not readonly
-                    memset(buf + proc_pos - done_pos + pos, 0, bs->journal.block_size - pos);
-                    bs->journal.next_free = prev_free;
-                    init_write_buf = buf + proc_pos - done_pos;
-                    init_write_sector = proc_pos;
-                    return 0;
-                }
-                auto clean_it = bs->clean_db.find(je->small_write.oid);
-                if (clean_it == bs->clean_db.end() ||
-                    clean_it->second.version < je->big_write.version)
-                {
-                    obj_ver_id ov = {
-                        .oid = je->small_write.oid,
-                        .version = je->small_write.version,
-                    };
-                    bs->dirty_db.emplace(ov, (dirty_entry){
-                        .state = ST_J_SYNCED,
-                        .flags = 0,
-                        .location = location,
-                        .offset = je->small_write.offset,
-                        .len = je->small_write.len,
-                        .journal_sector = proc_pos,
-                    });
-                    bs->journal.used_sectors[proc_pos]++;
-#ifdef BLOCKSTORE_DEBUG
-                    printf("journal offset %lu is used by %lu:%lu v%lu\n", proc_pos, ov.oid.inode, ov.oid.stripe, ov.version);
-#endif
-                    auto & unstab = bs->unstable_writes[ov.oid];
-                    unstab = unstab < ov.version ? ov.version : unstab;
-                }
-            }
-            else if (je->type == JE_BIG_WRITE)
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("je_big_write oid=%lu:%lu ver=%lu loc=%lu\n", je->big_write.oid.inode, je->big_write.oid.stripe, je->big_write.version, je->big_write.location);
-#endif
-                auto clean_it = bs->clean_db.find(je->big_write.oid);
-                if (clean_it == bs->clean_db.end() ||
-                    clean_it->second.version < je->big_write.version)
-                {
-                    // oid, version, block
-                    obj_ver_id ov = {
-                        .oid = je->big_write.oid,
-                        .version = je->big_write.version,
-                    };
-                    bs->dirty_db.emplace(ov, (dirty_entry){
-                        .state = ST_D_META_SYNCED,
-                        .flags = 0,
-                        .location = je->big_write.location,
-                        .offset = je->big_write.offset,
-                        .len = je->big_write.len,
-                        .journal_sector = proc_pos,
-                    });
-#ifdef BLOCKSTORE_DEBUG
-                    printf("Allocate block %lu\n", je->big_write.location >> bs->block_order);
-#endif
-                    bs->data_alloc->set(je->big_write.location >> bs->block_order, true);
-                    bs->journal.used_sectors[proc_pos]++;
-                    auto & unstab = bs->unstable_writes[ov.oid];
-                    unstab = unstab < ov.version ? ov.version : unstab;
-                }
-            }
-            else if (je->type == JE_STABLE)
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("je_stable oid=%lu:%lu ver=%lu\n", je->stable.oid.inode, je->stable.oid.stripe, je->stable.version);
-#endif
-                // oid, version
-                obj_ver_id ov = {
-                    .oid = je->stable.oid,
-                    .version = je->stable.version,
-                };
-                auto it = bs->dirty_db.find(ov);
-                if (it == bs->dirty_db.end())
-                {
-                    // journal contains a legitimate STABLE entry for a non-existing dirty write
-                    // this probably means that journal was trimmed between WRITE and STABLE entries
-                    // skip it
-                }
-                else
-                {
-                    while (1)
-                    {
-                        it->second.state = (it->second.state == ST_D_META_SYNCED
-                            ? ST_D_STABLE
-                            : (it->second.state == ST_DEL_SYNCED ? ST_DEL_STABLE : ST_J_STABLE));
-                        if (it == bs->dirty_db.begin())
-                            break;
-                        it--;
-                        if (it->first.oid != ov.oid || IS_STABLE(it->second.state))
-                            break;
-                    }
-                    bs->flusher->enqueue_flush(ov);
-                }
-                auto unstab_it = bs->unstable_writes.find(ov.oid);
-                if (unstab_it != bs->unstable_writes.end() && unstab_it->second <= ov.version)
-                {
-                    bs->unstable_writes.erase(unstab_it);
-                }
-            }
-            else if (je->type == JE_ROLLBACK)
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("je_rollback oid=%lu:%lu ver=%lu\n", je->rollback.oid.inode, je->rollback.oid.stripe, je->rollback.version);
-#endif
-                // rollback dirty writes of <oid> up to <version>
-                auto it = bs->dirty_db.lower_bound((obj_ver_id){
-                    .oid = je->rollback.oid,
-                    .version = UINT64_MAX,
-                });
-                if (it != bs->dirty_db.begin())
-                {
-                    uint64_t max_unstable = 0;
-                    auto rm_start = it;
-                    auto rm_end = it;
-                    it--;
-                    while (it->first.oid == je->rollback.oid &&
-                        it->first.version > je->rollback.version &&
-                        !IS_IN_FLIGHT(it->second.state) &&
-                        !IS_STABLE(it->second.state))
-                    {
-                        if (it->first.oid != je->rollback.oid)
-                            break;
-                        else if (it->first.version <= je->rollback.version)
-                        {
-                            if (!IS_STABLE(it->second.state))
-                                max_unstable = it->first.version;
-                            break;
-                        }
-                        else if (IS_STABLE(it->second.state))
-                            break;
-                        // Remove entry
-                        rm_start = it;
-                        if (it == bs->dirty_db.begin())
-                            break;
-                        it--;
-                    }
-                    if (rm_start != rm_end)
-                    {
-                        bs->erase_dirty(rm_start, rm_end, UINT64_MAX);
-                    }
-                    auto unstab_it = bs->unstable_writes.find(je->rollback.oid);
-                    if (unstab_it != bs->unstable_writes.end())
-                    {
-                        if (max_unstable == 0)
-                            bs->unstable_writes.erase(unstab_it);
-                        else
-                            unstab_it->second = max_unstable;
-                    }
-                }
-            }
-            else if (je->type == JE_DELETE)
-            {
-#ifdef BLOCKSTORE_DEBUG
-                printf("je_delete oid=%lu:%lu ver=%lu\n", je->del.oid.inode, je->del.oid.stripe, je->del.version);
-#endif
-                // oid, version
-                obj_ver_id ov = {
-                    .oid = je->del.oid,
-                    .version = je->del.version,
-                };
-                bs->dirty_db.emplace(ov, (dirty_entry){
-                    .state = ST_DEL_SYNCED,
-                    .flags = 0,
-                    .location = 0,
-                    .offset = 0,
-                    .len = 0,
-                    .journal_sector = proc_pos,
-                });
-                bs->journal.used_sectors[proc_pos]++;
-            }
-            started = true;
-            pos += je->size;
-            crc32_last = je->crc32;
-            entries_loaded++;
-        }
-    }
-    bs->journal.next_free = next_free;
-    return 1;
-}

blockstore_journal.cpp

@@ -1,186 +0,0 @@
-#include "blockstore_impl.h"
-
-blockstore_journal_check_t::blockstore_journal_check_t(blockstore_impl_t *bs)
-{
-    this->bs = bs;
-    sectors_required = 0;
-    next_pos = bs->journal.next_free;
-    next_sector = bs->journal.cur_sector;
-    next_in_pos = bs->journal.in_sector_pos;
-    right_dir = next_pos >= bs->journal.used_start;
-}
-
-// Check if we can write <required> entries of <size> bytes and <data_after> data bytes after them to the journal
-int blockstore_journal_check_t::check_available(blockstore_op_t *op, int required, int size, int data_after)
-{
-    while (1)
-    {
-        int fits = (bs->journal.block_size - next_in_pos) / size;
-        if (fits > 0)
-        {
-            required -= fits;
-            next_in_pos += fits * size;
-            sectors_required++;
-        }
-        else if (bs->journal.sector_info[next_sector].dirty)
-        {
-            // sectors_required is more like "sectors to write"
-            sectors_required++;
-        }
-        if (required <= 0)
-        {
-            break;
-        }
-        next_pos = next_pos + bs->journal.block_size;
-        if (next_pos >= bs->journal.len)
-        {
-            next_pos = bs->journal.block_size;
-            right_dir = false;
-        }
-        next_in_pos = 0;
-        if (bs->journal.sector_info[next_sector].usage_count > 0 ||
-            bs->journal.sector_info[next_sector].dirty)
-        {
-            next_sector = ((next_sector + 1) % bs->journal.sector_count);
-        }
-        if (bs->journal.sector_info[next_sector].usage_count > 0 ||
-            bs->journal.sector_info[next_sector].dirty)
-        {
-            // No memory buffer available. Wait for it.
-#ifdef BLOCKSTORE_DEBUG
-            printf("next journal buffer %d is still dirty=%d used=%d\n", next_sector,
-                bs->journal.sector_info[next_sector].dirty, bs->journal.sector_info[next_sector].usage_count);
-#endif
-            PRIV(op)->wait_for = WAIT_JOURNAL_BUFFER;
-            return 0;
-        }
-    }
-    if (data_after > 0)
-    {
-        next_pos = next_pos + data_after;
-        if (next_pos > bs->journal.len)
-        {
-            next_pos = bs->journal.block_size + data_after;
-            right_dir = false;
-        }
-    }
-    if (!right_dir && next_pos >= bs->journal.used_start-bs->journal.block_size)
-    {
-        // No space in the journal. Wait until used_start changes.
-        printf(
-            "Ran out of journal space (free space: %lu bytes)\n",
-            (bs->journal.next_free >= bs->journal.used_start
-                ? bs->journal.len-bs->journal.block_size - (bs->journal.next_free-bs->journal.used_start)
-                : bs->journal.used_start - bs->journal.next_free)
-        );
-        PRIV(op)->wait_for = WAIT_JOURNAL;
-        bs->flusher->force_start();
-        PRIV(op)->wait_detail = bs->journal.used_start;
-        return 0;
-    }
-    return 1;
-}
-
-journal_entry* prefill_single_journal_entry(journal_t & journal, uint16_t type, uint32_t size)
-{
-    if (journal.block_size - journal.in_sector_pos < size)
-    {
-        assert(!journal.sector_info[journal.cur_sector].dirty);
-        // Move to the next journal sector
-        if (journal.sector_info[journal.cur_sector].usage_count > 0)
-        {
-            // Also select next sector buffer in memory
-            journal.cur_sector = ((journal.cur_sector + 1) % journal.sector_count);
-        }
-        journal.sector_info[journal.cur_sector].offset = journal.next_free;
-        journal.in_sector_pos = 0;
-        journal.next_free = (journal.next_free+journal.block_size) < journal.len ? journal.next_free + journal.block_size : journal.block_size;
-        memset(journal.inmemory
-            ? journal.buffer + journal.sector_info[journal.cur_sector].offset
-            : journal.sector_buf + journal.block_size*journal.cur_sector, 0, journal.block_size);
-    }
-    journal_entry *je = (struct journal_entry*)(
-        (journal.inmemory
-            ? journal.buffer + journal.sector_info[journal.cur_sector].offset
-            : journal.sector_buf + journal.block_size*journal.cur_sector) + journal.in_sector_pos
-    );
-    journal.in_sector_pos += size;
-    je->magic = JOURNAL_MAGIC;
-    je->type = type;
-    je->size = size;
-    je->crc32_prev = journal.crc32_last;
-    journal.sector_info[journal.cur_sector].dirty = true;
-    return je;
-}
-
-void prepare_journal_sector_write(journal_t & journal, int cur_sector, io_uring_sqe *sqe, std::function<void(ring_data_t*)> cb)
-{
-    journal.sector_info[cur_sector].dirty = false;
-    journal.sector_info[cur_sector].usage_count++;
-    ring_data_t *data = ((ring_data_t*)sqe->user_data);
-    data->iov = (struct iovec){
-        (journal.inmemory
-            ? journal.buffer + journal.sector_info[cur_sector].offset
-            : journal.sector_buf + journal.block_size*cur_sector),
-        journal.block_size
-    };
-    data->callback = cb;
-    my_uring_prep_writev(
-        sqe, journal.fd, &data->iov, 1, journal.offset + journal.sector_info[cur_sector].offset
-    );
-}
-
-journal_t::~journal_t()
-{
-    if (sector_buf)
-        free(sector_buf);
-    if (sector_info)
-        free(sector_info);
-    if (buffer)
-        free(buffer);
-    sector_buf = NULL;
-    sector_info = NULL;
-    buffer = NULL;
-}
-
-bool journal_t::trim()
-{
-    auto journal_used_it = used_sectors.lower_bound(used_start);
-#ifdef BLOCKSTORE_DEBUG
-    printf(
-        "Trimming journal (used_start=%lu, next_free=%lu, first_used=%lu, usage_count=%lu)\n",
-        used_start, next_free,
-        journal_used_it == used_sectors.end() ? 0 : journal_used_it->first,
-        journal_used_it == used_sectors.end() ? 0 : journal_used_it->second
-    );
-#endif
-    if (journal_used_it == used_sectors.end())
-    {
-        // Journal is cleared to its end, restart from the beginning
-        journal_used_it = used_sectors.begin();
-        if (journal_used_it == used_sectors.end())
-        {
-            // Journal is empty
-            used_start = next_free;
-        }
-        else
-        {
-            used_start = journal_used_it->first;
-            // next_free does not need updating here
-        }
-    }
-    else if (journal_used_it->first > used_start)
-    {
-        // Journal is cleared up to <journal_used_it>
-        used_start = journal_used_it->first;
-    }
-    else
-    {
-        // Can't trim journal
-        return false;
-    }
-#ifdef BLOCKSTORE_DEBUG
-    printf("Journal trimmed to %lu (next_free=%lu)\n", used_start, next_free);
-#endif
-    return true;
-}

blockstore_open.cpp

@@ -1,317 +0,0 @@
-#include "blockstore_impl.h"
-
-static uint32_t is_power_of_two(uint64_t value)
-{
-    uint32_t l = 0;
-    while (value > 1)
-    {
-        if (value & 1)
-        {
-            return 64;
-        }
-        value = value >> 1;
-        l++;
-    }
-    return l;
-}
-
-void blockstore_impl_t::parse_config(blockstore_config_t & config)
-{
-    // Parse
-    if (config["readonly"] == "true" || config["readonly"] == "1" || config["readonly"] == "yes")
-    {
-        readonly = true;
-    }
-    if (config["disable_data_fsync"] == "true" || config["disable_data_fsync"] == "1" || config["disable_data_fsync"] == "yes")
-    {
-        disable_data_fsync = true;
-    }
-    if (config["disable_meta_fsync"] == "true" || config["disable_meta_fsync"] == "1" || config["disable_meta_fsync"] == "yes")
-    {
-        disable_meta_fsync = true;
-    }
-    if (config["disable_journal_fsync"] == "true" || config["disable_journal_fsync"] == "1" || config["disable_journal_fsync"] == "yes")
-    {
-        disable_journal_fsync = true;
-    }
-    metadata_buf_size = strtoull(config["meta_buf_size"].c_str(), NULL, 10);
-    cfg_journal_size = strtoull(config["journal_size"].c_str(), NULL, 10);
-    data_device = config["data_device"];
-    data_offset = strtoull(config["data_offset"].c_str(), NULL, 10);
-    meta_device = config["meta_device"];
-    meta_offset = strtoull(config["meta_offset"].c_str(), NULL, 10);
-    block_size = strtoull(config["block_size"].c_str(), NULL, 10);
-    inmemory_meta = config["inmemory_metadata"] != "false";
-    journal_device = config["journal_device"];
-    journal.offset = strtoull(config["journal_offset"].c_str(), NULL, 10);
-    journal.sector_count = strtoull(config["journal_sector_buffer_count"].c_str(), NULL, 10);
-    journal.inmemory = config["inmemory_journal"] != "false";
-    disk_alignment = strtoull(config["disk_alignment"].c_str(), NULL, 10);
-    journal_block_size = strtoull(config["journal_block_size"].c_str(), NULL, 10);
-    meta_block_size = strtoull(config["meta_block_size"].c_str(), NULL, 10);
-    bitmap_granularity = strtoull(config["bitmap_granularity"].c_str(), NULL, 10);
-    flusher_count = strtoull(config["flusher_count"].c_str(), NULL, 10);
-    // Validate
-    if (!block_size)
-    {
-        block_size = (1 << DEFAULT_ORDER);
-    }
-    if ((block_order = is_power_of_two(block_size)) >= 64 || block_size < MIN_BLOCK_SIZE || block_size >= MAX_BLOCK_SIZE)
-    {
-        throw std::runtime_error("Bad block size");
-    }
-    if (!flusher_count)
-    {
-        flusher_count = 32;
-    }
-    if (!disk_alignment)
-    {
-        disk_alignment = 512;
-    }
-    else if (disk_alignment % MEM_ALIGNMENT)
-    {
-        throw std::runtime_error("disk_alingment must be a multiple of "+std::to_string(MEM_ALIGNMENT));
-    }
-    if (!journal_block_size)
-    {
-        journal_block_size = 512;
-    }
-    else if (journal_block_size % MEM_ALIGNMENT)
-    {
-        throw std::runtime_error("journal_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
-    }
-    if (!meta_block_size)
-    {
-        meta_block_size = 512;
-    }
-    else if (meta_block_size % MEM_ALIGNMENT)
-    {
-        throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
-    }
-    if (data_offset % disk_alignment)
-    {
-        throw std::runtime_error("data_offset must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
-    }
-    if (!bitmap_granularity)
-    {
-        bitmap_granularity = 4096;
-    }
-    else if (bitmap_granularity % disk_alignment)
-    {
-        throw std::runtime_error("Sparse write tracking granularity must be a multiple of disk_alignment = "+std::to_string(disk_alignment));
-    }
-    if (block_size % bitmap_granularity)
-    {
-        throw std::runtime_error("Block size must be a multiple of sparse write tracking granularity");
-    }
-    if (journal_device == meta_device || meta_device == "" && journal_device == data_device)
-    {
-        journal_device = "";
-    }
-    if (meta_device == data_device)
-    {
-        meta_device = "";
-    }
-    if (meta_offset % meta_block_size)
-    {
-        throw std::runtime_error("meta_offset must be a multiple of meta_block_size = "+std::to_string(meta_block_size));
-    }
-    if (journal.offset % journal_block_size)
-    {
-        throw std::runtime_error("journal_offset must be a multiple of journal_block_size = "+std::to_string(journal_block_size));
-    }
-    if (journal.sector_count < 2)
-    {
-        journal.sector_count = 32;
-    }
-    if (metadata_buf_size < 65536)
-    {
-        metadata_buf_size = 4*1024*1024;
-    }
-    // init some fields
-    clean_entry_bitmap_size = block_size / bitmap_granularity / 8;
-    clean_entry_size = sizeof(clean_disk_entry) + clean_entry_bitmap_size;
-    journal.block_size = journal_block_size;
-    journal.next_free = journal_block_size;
-    journal.used_start = journal_block_size;
-    // no free space because sector is initially unmapped
-    journal.in_sector_pos = journal_block_size;
-}
-
-void blockstore_impl_t::calc_lengths()
-{
-    // data
-    data_len = data_size - data_offset;
-    if (data_fd == meta_fd && data_offset < meta_offset)
-    {
-        data_len = meta_offset - data_offset;
-    }
-    if (data_fd == journal.fd && data_offset < journal.offset)
-    {
-        data_len = data_len < journal.offset-data_offset
-            ? data_len : journal.offset-data_offset;
-    }
-    // meta
-    meta_area = (meta_fd == data_fd ? data_size : meta_size) - meta_offset;
-    if (meta_fd == data_fd && meta_offset <= data_offset)
-    {
-        meta_area = data_offset - meta_offset;
-    }
-    if (meta_fd == journal.fd && meta_offset <= journal.offset)
-    {
-        meta_area = meta_area < journal.offset-meta_offset
-            ? meta_area : journal.offset-meta_offset;
-    }
-    // journal
-    journal.len = (journal.fd == data_fd ? data_size : (journal.fd == meta_fd ? meta_size : journal.device_size)) - journal.offset;
-    if (journal.fd == data_fd && journal.offset <= data_offset)
-    {
-        journal.len = data_offset - journal.offset;
-    }
-    if (journal.fd == meta_fd && journal.offset <= meta_offset)
-    {
-        journal.len = journal.len < meta_offset-journal.offset
-            ? journal.len : meta_offset-journal.offset;
-    }
-    // required metadata size
-    block_count = data_len / block_size;
-    meta_len = ((block_count - 1 + meta_block_size / clean_entry_size) / (meta_block_size / clean_entry_size)) * meta_block_size;
-    if (meta_area < meta_len)
-    {
-        throw std::runtime_error("Metadata area is too small, need at least "+std::to_string(meta_len)+" bytes");
-    }
-    if (inmemory_meta)
-    {
-        metadata_buffer = memalign(MEM_ALIGNMENT, meta_len);
-        if (!metadata_buffer)
-            throw std::runtime_error("Failed to allocate memory for the metadata");
-    }
-    else if (clean_entry_bitmap_size)
-    {
-        clean_bitmap = (uint8_t*)malloc(block_count * clean_entry_bitmap_size);
-        if (!clean_bitmap)
-            throw std::runtime_error("Failed to allocate memory for the metadata sparse write bitmap");
-    }
-    // requested journal size
-    if (cfg_journal_size > journal.len)
-    {
-        throw std::runtime_error("Requested journal_size is too large");
-    }
-    else if (cfg_journal_size > 0)
-    {
-        journal.len = cfg_journal_size;
-    }
-    if (journal.len < MIN_JOURNAL_SIZE)
-    {
-        throw std::runtime_error("Journal is too small, need at least "+std::to_string(MIN_JOURNAL_SIZE)+" bytes");
-    }
-    if (journal.inmemory)
-    {
-        journal.buffer = memalign(MEM_ALIGNMENT, journal.len);
-        if (!journal.buffer)
-            throw std::runtime_error("Failed to allocate memory for journal");
-    }
-}
-
-void check_size(int fd, uint64_t *size, std::string name)
-{
-    int sectsize;
-    struct stat st;
-    if (fstat(fd, &st) < 0)
-    {
-        throw std::runtime_error("Failed to stat "+name);
-    }
-    if (S_ISREG(st.st_mode))
-    {
-        *size = st.st_size;
-    }
-    else if (S_ISBLK(st.st_mode))
-    {
-        if (ioctl(fd, BLKSSZGET, &sectsize) < 0 ||
-            ioctl(fd, BLKGETSIZE64, size) < 0 ||
-            sectsize != 512)
-        {
-            throw std::runtime_error(name+" sector is not equal to 512 bytes");
-        }
-    }
-    else
-    {
-        throw std::runtime_error(name+" is neither a file nor a block device");
-    }
-}
-
-void blockstore_impl_t::open_data()
-{
-    data_fd = open(data_device.c_str(), O_DIRECT|O_RDWR);
-    if (data_fd == -1)
-    {
-        throw std::runtime_error("Failed to open data device");
-    }
-    check_size(data_fd, &data_size, "data device");
-    if (data_offset >= data_size)
-    {
-        throw std::runtime_error("data_offset exceeds device size = "+std::to_string(data_size));
-    }
-}
-
-void blockstore_impl_t::open_meta()
-{
-    if (meta_device != "")
-    {
-        meta_offset = 0;
-        meta_fd = open(meta_device.c_str(), O_DIRECT|O_RDWR);
-        if (meta_fd == -1)
-        {
-            throw std::runtime_error("Failed to open metadata device");
-        }
-        check_size(meta_fd, &meta_size, "metadata device");
-        if (meta_offset >= meta_size)
-        {
-            throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(meta_size));
-        }
-    }
-    else
-    {
-        meta_fd = data_fd;
-        disable_meta_fsync = disable_data_fsync;
-        meta_size = 0;
-        if (meta_offset >= data_size)
-        {
-            throw std::runtime_error("meta_offset exceeds device size = "+std::to_string(data_size));
-        }
-    }
-}
-
-void blockstore_impl_t::open_journal()
-{
-    if (journal_device != "")
-    {
-        journal.fd = open(journal_device.c_str(), O_DIRECT|O_RDWR);
-        if (journal.fd == -1)
-        {
-            throw std::runtime_error("Failed to open journal device");
-        }
-        check_size(journal.fd, &journal.device_size, "metadata device");
-    }
-    else
-    {
-        journal.fd = meta_fd;
-        disable_journal_fsync = disable_meta_fsync;
-        journal.device_size = 0;
-        if (journal.offset >= data_size)
-        {
-            throw std::runtime_error("journal_offset exceeds device size");
-        }
-    }
-    journal.sector_info = (journal_sector_info_t*)calloc(journal.sector_count, sizeof(journal_sector_info_t));
-    if (!journal.sector_info)
-    {
-        throw std::bad_alloc();
-    }
-    if (!journal.inmemory)
-    {
-        journal.sector_buf = (uint8_t*)memalign(MEM_ALIGNMENT, journal.sector_count * journal_block_size);
-        if (!journal.sector_buf)
-            throw std::bad_alloc();
-    }
-}

blockstore_read.cpp

@@ -1,235 +0,0 @@
-#include "blockstore_impl.h"
-
-int blockstore_impl_t::fulfill_read_push(blockstore_op_t *op, void *buf, uint64_t offset, uint64_t len,
-    uint32_t item_state, uint64_t item_version)
-{
-    if (!len)
-    {
-        // Zero-length version - skip
-        return 1;
-    }
-    if (IS_IN_FLIGHT(item_state))
-    {
-        // Pause until it's written somewhere
-        PRIV(op)->wait_for = WAIT_IN_FLIGHT;
-        PRIV(op)->wait_detail = item_version;
-        return 0;
-    }
-    else if (IS_DELETE(item_state))
-    {
-        // item is unallocated - return zeroes
-        memset(buf, 0, len);
-        return 1;
-    }
-    if (journal.inmemory && IS_JOURNAL(item_state))
-    {
-        memcpy(buf, journal.buffer + offset, len);
-        return 1;
-    }
-    BS_SUBMIT_GET_SQE(sqe, data);
-    data->iov = (struct iovec){ buf, len };
-    PRIV(op)->pending_ops++;
-    my_uring_prep_readv(
-        sqe,
-        IS_JOURNAL(item_state) ? journal.fd : data_fd,
-        &data->iov, 1,
-        (IS_JOURNAL(item_state) ? journal.offset : data_offset) + offset
-    );
-    data->callback = [this, op](ring_data_t *data) { handle_read_event(data, op); };
-    return 1;
-}
-
-int blockstore_impl_t::fulfill_read(blockstore_op_t *read_op, uint64_t &fulfilled, uint32_t item_start, uint32_t item_end,
-    uint32_t item_state, uint64_t item_version, uint64_t item_location)
-{
-    uint32_t cur_start = item_start;
-    if (cur_start < read_op->offset + read_op->len && item_end > read_op->offset)
-    {
-        cur_start = cur_start < read_op->offset ? read_op->offset : cur_start;
-        item_end = item_end > read_op->offset + read_op->len ? read_op->offset + read_op->len : item_end;
-        auto it = PRIV(read_op)->read_vec.begin();
-        while (1)
-        {
-            for (; it != PRIV(read_op)->read_vec.end(); it++)
-                if (it->offset >= cur_start)
-                    break;
-            if (it == PRIV(read_op)->read_vec.end() || it->offset > cur_start)
-            {
-                fulfill_read_t el = {
-                    .offset = cur_start,
-                    .len = it == PRIV(read_op)->read_vec.end() || it->offset >= item_end ? item_end-cur_start : it->offset-cur_start,
-                };
-                it = PRIV(read_op)->read_vec.insert(it, el);
-                if (!fulfill_read_push(read_op,
-                    read_op->buf + el.offset - read_op->offset,
-                    item_location + el.offset - item_start,
-                    el.len, item_state, item_version))
-                {
-                    return 0;
-                }
-                fulfilled += el.len;
-            }
-            cur_start = it->offset + it->len;
-            if (it == PRIV(read_op)->read_vec.end() || cur_start >= item_end)
-                break;
-        }
-    }
-    return 1;
-}
-
-int blockstore_impl_t::dequeue_read(blockstore_op_t *read_op)
-{
-    auto clean_it = clean_db.find(read_op->oid);
-    auto dirty_it = dirty_db.upper_bound((obj_ver_id){
-        .oid = read_op->oid,
-        .version = UINT64_MAX,
-    });
-    if (dirty_it != dirty_db.begin())
-        dirty_it--;
-    bool clean_found = clean_it != clean_db.end();
-    bool dirty_found = (dirty_it != dirty_db.end() && dirty_it->first.oid == read_op->oid);
-    if (!clean_found && !dirty_found)
-    {
-        // region is not allocated - return zeroes
-        memset(read_op->buf, 0, read_op->len);
-        read_op->version = 0;
-        read_op->retval = read_op->len;
-        FINISH_OP(read_op);
-        return 1;
-    }
-    uint64_t fulfilled = 0;
-    PRIV(read_op)->pending_ops = 0;
-    uint64_t result_version = 0;
-    if (dirty_found)
-    {
-        while (dirty_it->first.oid == read_op->oid)
-        {
-            dirty_entry& dirty = dirty_it->second;
-            bool version_ok = read_op->version >= dirty_it->first.version;
-            if (IS_SYNCED(dirty.state))
-            {
-                if (!version_ok && read_op->version != 0)
-                    read_op->version = dirty_it->first.version;
-                version_ok = true;
-            }
-            if (version_ok)
-            {
-                if (!result_version)
-                {
-                    result_version = dirty_it->first.version;
-                }
-                if (!fulfill_read(read_op, fulfilled, dirty.offset, dirty.offset + dirty.len,
-                    dirty.state, dirty_it->first.version, dirty.location + (IS_JOURNAL(dirty.state) ? 0 : dirty.offset)))
-                {
-                    // need to wait. undo added requests, don't dequeue op
-                    PRIV(read_op)->read_vec.clear();
-                    return 0;
-                }
-            }
-            if (fulfilled == read_op->len || dirty_it == dirty_db.begin())
-            {
-                break;
-            }
-            dirty_it--;
-        }
-    }
-    if (clean_it != clean_db.end() && fulfilled < read_op->len)
-    {
-        if (!result_version)
-        {
-            result_version = clean_it->second.version;
-        }
-        if (!clean_entry_bitmap_size)
-        {
-            if (!fulfill_read(read_op, fulfilled, 0, block_size, ST_CURRENT, 0, clean_it->second.location))
-            {
-                // need to wait. undo added requests, don't dequeue op
-                PRIV(read_op)->read_vec.clear();
-                return 0;
-            }
-        }
-        else
-        {
-            uint64_t meta_loc = clean_it->second.location >> block_order;
-            uint8_t *clean_entry_bitmap;
-            if (inmemory_meta)
-            {
-                uint64_t sector = (meta_loc / (meta_block_size / clean_entry_size)) * meta_block_size;
-                uint64_t pos = (meta_loc % (meta_block_size / clean_entry_size));
-                clean_entry_bitmap = (uint8_t*)(metadata_buffer + sector + pos*clean_entry_size + sizeof(clean_disk_entry));
-            }
-            else
-            {
-                clean_entry_bitmap = (uint8_t*)(clean_bitmap + meta_loc*clean_entry_bitmap_size);
-            }
-            uint64_t bmp_start = 0, bmp_end = 0, bmp_size = block_size/bitmap_granularity;
-            while (bmp_start < bmp_size)
-            {
-                while (!(clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7))) && bmp_end < bmp_size)
-                {
-                    bmp_end++;
-                }
-                if (bmp_end > bmp_start)
-                {
-                    // fill with zeroes
-                    fulfill_read(read_op, fulfilled, bmp_start * bitmap_granularity,
-                        bmp_end * bitmap_granularity, ST_DEL_STABLE, 0, 0);
-                }
-                bmp_start = bmp_end;
-                while (clean_entry_bitmap[bmp_end >> 3] & (1 << (bmp_end & 0x7)) && bmp_end < bmp_size)
-                {
-                    bmp_end++;
-                }
-                if (bmp_end > bmp_start)
-                {
-                    if (!fulfill_read(read_op, fulfilled, bmp_start * bitmap_granularity,
-                        bmp_end * bitmap_granularity, ST_CURRENT, 0, clean_it->second.location + bmp_start * bitmap_granularity))
-                    {
-                        // need to wait. undo added requests, don't dequeue op
-                        PRIV(read_op)->read_vec.clear();
-                        return 0;
-                    }
-                    bmp_start = bmp_end;
-                }
-            }
-        }
-    }
-    else if (fulfilled < read_op->len)
-    {
-        // fill remaining parts with zeroes
-        fulfill_read(read_op, fulfilled, 0, block_size, ST_DEL_STABLE, 0, 0);
-    }
-    assert(fulfilled == read_op->len);
-    read_op->version = result_version;
-    if (!PRIV(read_op)->pending_ops)
-    {
-        // everything is fulfilled from memory
-        if (!PRIV(read_op)->read_vec.size())
-        {
-            // region is not allocated - return zeroes
-            memset(read_op->buf, 0, read_op->len);
-        }
-        read_op->retval = read_op->len;
-        FINISH_OP(read_op);
-        return 1;
-    }
-    read_op->retval = 0;
-    return 1;
-}
-
-void blockstore_impl_t::handle_read_event(ring_data_t *data, blockstore_op_t *op)
-{
-    live = true;
-    PRIV(op)->pending_ops--;
-    if (data->res != data->iov.iov_len)
-    {
-        // read error
-        op->retval = data->res;
-    }
-    if (PRIV(op)->pending_ops == 0)
-    {
-        if (op->retval == 0)
-            op->retval = op->len;
-        FINISH_OP(op);
-    }
-}

blockstore_rollback.cpp

@@ -1,187 +0,0 @@
-#include "blockstore_impl.h"
-
-int blockstore_impl_t::dequeue_rollback(blockstore_op_t *op)
-{
-    obj_ver_id* v;
-    int i, todo = op->len;
-    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-    {
-        // Check that there are some versions greater than v->version (which may be zero),
-        // check that they're unstable, synced, and not currently written to
-        auto dirty_it = dirty_db.lower_bound((obj_ver_id){
-            .oid = v->oid,
-            .version = UINT64_MAX,
-        });
-        if (dirty_it == dirty_db.begin())
-        {
-        bad_op:
-            op->retval = -EINVAL;
-            FINISH_OP(op);
-            return 1;
-        }
-        else
-        {
-            dirty_it--;
-            if (dirty_it->first.oid != v->oid || dirty_it->first.version < v->version)
-            {
-                goto bad_op;
-            }
-            while (dirty_it->first.oid == v->oid && dirty_it->first.version > v->version)
-            {
-                if (!IS_SYNCED(dirty_it->second.state) ||
-                    IS_STABLE(dirty_it->second.state))
-                {
-                    goto bad_op;
-                }
-                if (dirty_it == dirty_db.begin())
-                {
-                    break;
-                }
-                dirty_it--;
-            }
-        }
-    }
-    // Check journal space
-    blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, todo, sizeof(journal_entry_rollback), 0))
-    {
-        return 0;
-    }
-    // There is sufficient space. Get SQEs
-    struct io_uring_sqe *sqe[space_check.sectors_required];
-    for (i = 0; i < space_check.sectors_required; i++)
-    {
-        BS_SUBMIT_GET_SQE_DECL(sqe[i]);
-    }
-    // Prepare and submit journal entries
-    auto cb = [this, op](ring_data_t *data) { handle_rollback_event(data, op); };
-    int s = 0, cur_sector = -1;
-    if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_rollback) &&
-        journal.sector_info[journal.cur_sector].dirty)
-    {
-        if (cur_sector == -1)
-            PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-        cur_sector = journal.cur_sector;
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-    }
-    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-    {
-        // FIXME This is here only for the purpose of tracking unstable_writes. Remove if not required
-        // FIXME ...aaaand this is similar to blockstore_init.cpp - maybe dedup it?
-        auto dirty_it = dirty_db.lower_bound((obj_ver_id){
-            .oid = v->oid,
-            .version = UINT64_MAX,
-        });
-        uint64_t max_unstable = 0;
-        while (dirty_it != dirty_db.begin())
-        {
-            dirty_it--;
-            if (dirty_it->first.oid != v->oid)
-                break;
-            else if (dirty_it->first.version <= v->version)
-            {
-                if (!IS_STABLE(dirty_it->second.state))
-                    max_unstable = dirty_it->first.version;
-                break;
-            }
-        }
-        auto unstab_it = unstable_writes.find(v->oid);
-        if (unstab_it != unstable_writes.end())
-        {
-            if (max_unstable == 0)
-                unstable_writes.erase(unstab_it);
-            else
-                unstab_it->second = max_unstable;
-        }
-        journal_entry_rollback *je = (journal_entry_rollback*)
-            prefill_single_journal_entry(journal, JE_ROLLBACK, sizeof(journal_entry_rollback));
-        journal.sector_info[journal.cur_sector].dirty = false;
-        je->oid = v->oid;
-        je->version = v->version;
-        je->crc32 = je_crc32((journal_entry*)je);
-        journal.crc32_last = je->crc32;
-        if (cur_sector != journal.cur_sector)
-        {
-            if (cur_sector == -1)
-                PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-            prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-        }
-    }
-    PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-    PRIV(op)->pending_ops = s;
-    return 1;
-}
-
-void blockstore_impl_t::handle_rollback_event(ring_data_t *data, blockstore_op_t *op)
-{
-    live = true;
-    if (data->res != data->iov.iov_len)
-    {
-        throw std::runtime_error(
-            "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
-    }
-    PRIV(op)->pending_ops--;
-    if (PRIV(op)->pending_ops == 0)
-    {
-        // Release used journal sectors
-        release_journal_sectors(op);
-        obj_ver_id* v;
-        int i;
-        for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-        {
-            // Erase dirty_db entries
-            auto rm_end = dirty_db.lower_bound((obj_ver_id){
-                .oid = v->oid,
-                .version = UINT64_MAX,
-            });
-            rm_end--;
-            auto rm_start = rm_end;
-            while (1)
-            {
-                if (rm_end->first.oid != v->oid)
-                    break;
-                else if (rm_end->first.version <= v->version)
-                    break;
-                rm_start = rm_end;
-                if (rm_end == dirty_db.begin())
-                    break;
-                rm_end--;
-            }
-            if (rm_end != rm_start)
-                erase_dirty(rm_start, rm_end, UINT64_MAX);
-        }
-        journal.trim();
-        // Acknowledge op
-        op->retval = 0;
-        FINISH_OP(op);
-    }
-}
-
-void blockstore_impl_t::erase_dirty(blockstore_dirty_db_t::iterator dirty_start, blockstore_dirty_db_t::iterator dirty_end, uint64_t clean_loc)
-{
-    auto dirty_it = dirty_end;
-    while (dirty_it != dirty_start)
-    {
-        dirty_it--;
-        if (IS_BIG_WRITE(dirty_it->second.state) && dirty_it->second.location != clean_loc)
-        {
-#ifdef BLOCKSTORE_DEBUG
-            printf("Free block %lu\n", dirty_it->second.location >> block_order);
-#endif
-            data_alloc->set(dirty_it->second.location >> block_order, false);
-        }
-#ifdef BLOCKSTORE_DEBUG
-        printf("remove usage of journal offset %lu by %lu:%lu v%lu\n", dirty_it->second.journal_sector,
-            dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version);
-#endif
-        int used = --journal.used_sectors[dirty_it->second.journal_sector];
-        if (used == 0)
-        {
-            journal.used_sectors.erase(dirty_it->second.journal_sector);
-        }
-    }
-    dirty_db.erase(dirty_start, dirty_end);
-}

blockstore_stable.cpp

@@ -1,195 +0,0 @@
-#include "blockstore_impl.h"
-
-// Stabilize small write:
-// 1) Copy data from the journal to the data device
-// 2) Increase version on the metadata device and sync it
-// 3) Advance clean_db entry's version, clear previous journal entries
-//
-// This makes 1 4K small write+sync look like:
-// 512b+4K (journal) + sync + 512b (journal) + sync + 4K (data) [+ sync?] + 512b (metadata) + sync.
-// WA = 2.375. It's not the best, SSD FTL-like redirect-write could probably be lower
-// even with defragmentation. But it's fixed and it's still better than in Ceph. :)
-// except for HDD-only clusters, because each write results in 3 seeks.
-
-// Stabilize big write:
-// 1) Copy metadata from the journal to the metadata device
-// 2) Move dirty_db entry to clean_db and clear previous journal entries
-//
-// This makes 1 128K big write+sync look like:
-// 128K (data) + sync + 512b (journal) + sync + 512b (journal) + sync + 512b (metadata) + sync.
-// WA = 1.012. Very good :)
-
-// Stabilize delete:
-// 1) Remove metadata entry and sync it
-// 2) Remove dirty_db entry and clear previous journal entries
-// We have 2 problems here:
-// - In the cluster environment, we must store the "tombstones" of deleted objects until
-//   all replicas (not just quorum) agrees about their deletion. That is, "stabilize" is
-//   not possible for deletes in degraded placement groups
-// - With simple "fixed" metadata tables we can't just clear the metadata entry of the latest
-//   object version. We must clear all previous entries, too.
-// FIXME Fix both problems - probably, by switching from "fixed" metadata tables to "dynamic"
-
-// AND We must do it in batches, for the sake of reduced fsync call count
-// AND We must know what we stabilize. Basic workflow is like:
-// 1) primary OSD receives sync request
-// 2) it submits syncs to blockstore and peers
-// 3) after everyone acks sync it acks sync to the client
-// 4) after a while it takes his synced object list and sends stabilize requests
-//    to peers and to its own blockstore, thus freeing the old version
-
-int blockstore_impl_t::dequeue_stable(blockstore_op_t *op)
-{
-    obj_ver_id* v;
-    int i, todo = 0;
-    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-    {
-        auto dirty_it = dirty_db.find(*v);
-        if (dirty_it == dirty_db.end())
-        {
-            auto clean_it = clean_db.find(v->oid);
-            if (clean_it == clean_db.end() || clean_it->second.version < v->version)
-            {
-                // No such object version
-                op->retval = -EINVAL;
-                FINISH_OP(op);
-                return 1;
-            }
-            else
-            {
-                // Already stable
-            }
-        }
-        else if (IS_UNSYNCED(dirty_it->second.state))
-        {
-            // Object not synced yet. Caller must sync it first
-            op->retval = EAGAIN;
-            FINISH_OP(op);
-            return 1;
-        }
-        else if (!IS_STABLE(dirty_it->second.state))
-        {
-            todo++;
-        }
-    }
-    if (!todo)
-    {
-        // Already stable
-        op->retval = 0;
-        FINISH_OP(op);
-        return 1;
-    }
-    // Check journal space
-    blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, todo, sizeof(journal_entry_stable), 0))
-    {
-        return 0;
-    }
-    // There is sufficient space. Get SQEs
-    struct io_uring_sqe *sqe[space_check.sectors_required];
-    for (i = 0; i < space_check.sectors_required; i++)
-    {
-        BS_SUBMIT_GET_SQE_DECL(sqe[i]);
-    }
-    // Prepare and submit journal entries
-    auto cb = [this, op](ring_data_t *data) { handle_stable_event(data, op); };
-    int s = 0, cur_sector = -1;
-    if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_stable) &&
-        journal.sector_info[journal.cur_sector].dirty)
-    {
-        if (cur_sector == -1)
-            PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-        cur_sector = journal.cur_sector;
-        prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-    }
-    for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-    {
-        auto unstab_it = unstable_writes.find(v->oid);
-        if (unstab_it != unstable_writes.end() &&
-            unstab_it->second <= v->version)
-        {
-            unstable_writes.erase(unstab_it);
-        }
-        journal_entry_stable *je = (journal_entry_stable*)
-            prefill_single_journal_entry(journal, JE_STABLE, sizeof(journal_entry_stable));
-        journal.sector_info[journal.cur_sector].dirty = false;
-        je->oid = v->oid;
-        je->version = v->version;
-        je->crc32 = je_crc32((journal_entry*)je);
-        journal.crc32_last = je->crc32;
-        if (cur_sector != journal.cur_sector)
-        {
-            if (cur_sector == -1)
-                PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-            prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-        }
-    }
-    PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-    PRIV(op)->pending_ops = s;
-    return 1;
-}
-
-void blockstore_impl_t::handle_stable_event(ring_data_t *data, blockstore_op_t *op)
-{
-    live = true;
-    if (data->res != data->iov.iov_len)
-    {
-        throw std::runtime_error(
-            "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
-    }
-    PRIV(op)->pending_ops--;
-    if (PRIV(op)->pending_ops == 0)
-    {
-        // Release used journal sectors
-        release_journal_sectors(op);
-        // Mark dirty_db entries as stable, acknowledge op completion
-        obj_ver_id* v;
-        int i;
-        for (i = 0, v = (obj_ver_id*)op->buf; i < op->len; i++, v++)
-        {
-            // Mark all dirty_db entries up to op->version as stable
-            auto dirty_it = dirty_db.find(*v);
-            if (dirty_it != dirty_db.end())
-            {
-                while (1)
-                {
-                    if (dirty_it->second.state == ST_J_SYNCED)
-                    {
-                        dirty_it->second.state = ST_J_STABLE;
-                    }
-                    else if (dirty_it->second.state == ST_D_META_SYNCED)
-                    {
-                        dirty_it->second.state = ST_D_STABLE;
-                    }
-                    else if (dirty_it->second.state == ST_DEL_SYNCED)
-                    {
-                        dirty_it->second.state = ST_DEL_STABLE;
-                    }
-                    else if (IS_STABLE(dirty_it->second.state))
-                    {
-                        break;
-                    }
-                    if (dirty_it == dirty_db.begin())
-                    {
-                        break;
-                    }
-                    dirty_it--;
-                    if (dirty_it->first.oid != v->oid)
-                    {
-                        break;
-                    }
-                }
-#ifdef BLOCKSTORE_DEBUG
-                printf("enqueue_flush %lu:%lu v%lu\n", v->oid.inode, v->oid.stripe, v->version);
-#endif
-                flusher->enqueue_flush(*v);
-            }
-        }
-        // Acknowledge op
-        op->retval = 0;
-        FINISH_OP(op);
-    }
-}

blockstore_sync.cpp

@@ -1,269 +0,0 @@
-#include "blockstore_impl.h"
-
-#define SYNC_HAS_SMALL 1
-#define SYNC_HAS_BIG 2
-#define SYNC_DATA_SYNC_SENT 3
-#define SYNC_DATA_SYNC_DONE 4
-#define SYNC_JOURNAL_WRITE_SENT 5
-#define SYNC_JOURNAL_WRITE_DONE 6
-#define SYNC_JOURNAL_SYNC_SENT 7
-#define SYNC_DONE 8
-
-int blockstore_impl_t::dequeue_sync(blockstore_op_t *op)
-{
-    if (PRIV(op)->sync_state == 0)
-    {
-        stop_sync_submitted = false;
-        PRIV(op)->sync_big_writes.swap(unsynced_big_writes);
-        PRIV(op)->sync_small_writes.swap(unsynced_small_writes);
-        PRIV(op)->sync_small_checked = 0;
-        PRIV(op)->sync_big_checked = 0;
-        unsynced_big_writes.clear();
-        unsynced_small_writes.clear();
-        if (PRIV(op)->sync_big_writes.size() > 0)
-            PRIV(op)->sync_state = SYNC_HAS_BIG;
-        else if (PRIV(op)->sync_small_writes.size() > 0)
-            PRIV(op)->sync_state = SYNC_HAS_SMALL;
-        else
-            PRIV(op)->sync_state = SYNC_DONE;
-        // Always add sync to in_progress_syncs because we clear unsynced_big_writes and unsynced_small_writes
-        PRIV(op)->prev_sync_count = in_progress_syncs.size();
-        PRIV(op)->in_progress_ptr = in_progress_syncs.insert(in_progress_syncs.end(), op);
-    }
-    continue_sync(op);
-    // Always dequeue because we always add syncs to in_progress_syncs
-    return 1;
-}
-
-int blockstore_impl_t::continue_sync(blockstore_op_t *op)
-{
-    auto cb = [this, op](ring_data_t *data) { handle_sync_event(data, op); };
-    if (PRIV(op)->sync_state == SYNC_HAS_SMALL)
-    {
-        // No big writes, just fsync the journal
-        for (; PRIV(op)->sync_small_checked < PRIV(op)->sync_small_writes.size(); PRIV(op)->sync_small_checked++)
-        {
-            if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_small_writes[PRIV(op)->sync_small_checked]].state))
-            {
-                // Wait for small inflight writes to complete
-                return 0;
-            }
-        }
-        if (journal.sector_info[journal.cur_sector].dirty)
-        {
-            // Write out the last journal sector if it happens to be dirty
-            BS_SUBMIT_GET_ONLY_SQE(sqe);
-            prepare_journal_sector_write(journal, journal.cur_sector, sqe, cb);
-            PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-            PRIV(op)->pending_ops = 1;
-            PRIV(op)->sync_state = SYNC_JOURNAL_WRITE_SENT;
-            return 1;
-        }
-        else
-        {
-            PRIV(op)->sync_state = SYNC_JOURNAL_WRITE_DONE;
-        }
-    }
-    if (PRIV(op)->sync_state == SYNC_HAS_BIG)
-    {
-        for (; PRIV(op)->sync_big_checked < PRIV(op)->sync_big_writes.size(); PRIV(op)->sync_big_checked++)
-        {
-            if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_big_writes[PRIV(op)->sync_big_checked]].state))
-            {
-                // Wait for big inflight writes to complete
-                return 0;
-            }
-        }
-        // 1st step: fsync data
-        if (!disable_data_fsync)
-        {
-            BS_SUBMIT_GET_SQE(sqe, data);
-            my_uring_prep_fsync(sqe, data_fd, IORING_FSYNC_DATASYNC);
-            data->iov = { 0 };
-            data->callback = cb;
-            PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 0;
-            PRIV(op)->pending_ops = 1;
-            PRIV(op)->sync_state = SYNC_DATA_SYNC_SENT;
-            return 1;
-        }
-        else
-        {
-            PRIV(op)->sync_state = SYNC_DATA_SYNC_DONE;
-        }
-    }
-    if (PRIV(op)->sync_state == SYNC_DATA_SYNC_DONE)
-    {
-        for (; PRIV(op)->sync_small_checked < PRIV(op)->sync_small_writes.size(); PRIV(op)->sync_small_checked++)
-        {
-            if (IS_IN_FLIGHT(dirty_db[PRIV(op)->sync_small_writes[PRIV(op)->sync_small_checked]].state))
-            {
-                // Wait for small inflight writes to complete
-                return 0;
-            }
-        }
-        // 2nd step: Data device is synced, prepare & write journal entries
-        // Check space in the journal and journal memory buffers
-        blockstore_journal_check_t space_check(this);
-        if (!space_check.check_available(op, PRIV(op)->sync_big_writes.size(), sizeof(journal_entry_big_write), 0))
-        {
-            return 0;
-        }
-        // Get SQEs. Don't bother about merging, submit each journal sector as a separate request
-        struct io_uring_sqe *sqe[space_check.sectors_required];
-        for (int i = 0; i < space_check.sectors_required; i++)
-        {
-            BS_SUBMIT_GET_SQE_DECL(sqe[i]);
-        }
-        // Prepare and submit journal entries
-        auto it = PRIV(op)->sync_big_writes.begin();
-        int s = 0, cur_sector = -1;
-        if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_big_write) &&
-            journal.sector_info[journal.cur_sector].dirty)
-        {
-            if (cur_sector == -1)
-                PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-            cur_sector = journal.cur_sector;
-            prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-        }
-        while (it != PRIV(op)->sync_big_writes.end())
-        {
-            journal_entry_big_write *je = (journal_entry_big_write*)
-                prefill_single_journal_entry(journal, JE_BIG_WRITE, sizeof(journal_entry_big_write));
-            dirty_db[*it].journal_sector = journal.sector_info[journal.cur_sector].offset;
-            journal.sector_info[journal.cur_sector].dirty = false;
-            journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
-#ifdef BLOCKSTORE_DEBUG
-            printf("journal offset %lu is used by %lu:%lu v%lu\n", dirty_db[*it].journal_sector, it->oid.inode, it->oid.stripe, it->version);
-#endif
-            je->oid = it->oid;
-            je->version = it->version;
-            je->offset = dirty_db[*it].offset;
-            je->len = dirty_db[*it].len;
-            je->location = dirty_db[*it].location;
-            je->crc32 = je_crc32((journal_entry*)je);
-            journal.crc32_last = je->crc32;
-            it++;
-            if (cur_sector != journal.cur_sector)
-            {
-                if (cur_sector == -1)
-                    PRIV(op)->min_used_journal_sector = 1 + journal.cur_sector;
-                cur_sector = journal.cur_sector;
-                prepare_journal_sector_write(journal, cur_sector, sqe[s++], cb);
-            }
-        }
-        PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-        PRIV(op)->pending_ops = s;
-        PRIV(op)->sync_state = SYNC_JOURNAL_WRITE_SENT;
-        return 1;
-    }
-    if (PRIV(op)->sync_state == SYNC_JOURNAL_WRITE_DONE)
-    {
-        if (!disable_journal_fsync)
-        {
-            BS_SUBMIT_GET_SQE(sqe, data);
-            my_uring_prep_fsync(sqe, journal.fd, IORING_FSYNC_DATASYNC);
-            data->iov = { 0 };
-            data->callback = cb;
-            PRIV(op)->pending_ops = 1;
-            PRIV(op)->sync_state = SYNC_JOURNAL_SYNC_SENT;
-            return 1;
-        }
-        else
-        {
-            PRIV(op)->sync_state = SYNC_DONE;
-        }
-    }
-    if (PRIV(op)->sync_state == SYNC_DONE)
-    {
-        ack_sync(op);
-    }
-    return 1;
-}
-
-void blockstore_impl_t::handle_sync_event(ring_data_t *data, blockstore_op_t *op)
-{
-    live = true;
-    if (data->res != data->iov.iov_len)
-    {
-        throw std::runtime_error(
-            "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
-    }
-    PRIV(op)->pending_ops--;
-    if (PRIV(op)->pending_ops == 0)
-    {
-        // Release used journal sectors
-        release_journal_sectors(op);
-        // Handle states
-        if (PRIV(op)->sync_state == SYNC_DATA_SYNC_SENT)
-        {
-            PRIV(op)->sync_state = SYNC_DATA_SYNC_DONE;
-        }
-        else if (PRIV(op)->sync_state == SYNC_JOURNAL_WRITE_SENT)
-        {
-            PRIV(op)->sync_state = SYNC_JOURNAL_WRITE_DONE;
-        }
-        else if (PRIV(op)->sync_state == SYNC_JOURNAL_SYNC_SENT)
-        {
-            PRIV(op)->sync_state = SYNC_DONE;
-            ack_sync(op);
-        }
-        else
-        {
-            throw std::runtime_error("BUG: unexpected sync op state");
-        }
-    }
-}
-
-int blockstore_impl_t::ack_sync(blockstore_op_t *op)
-{
-    if (PRIV(op)->sync_state == SYNC_DONE && PRIV(op)->prev_sync_count == 0)
-    {
-        // Remove dependency of subsequent syncs
-        auto it = PRIV(op)->in_progress_ptr;
-        int done_syncs = 1;
-        ++it;
-        // Acknowledge sync
-        ack_one_sync(op);
-        while (it != in_progress_syncs.end())
-        {
-            auto & next_sync = *it++;
-            PRIV(next_sync)->prev_sync_count -= done_syncs;
-            if (PRIV(next_sync)->prev_sync_count == 0 && PRIV(next_sync)->sync_state == SYNC_DONE)
-            {
-                done_syncs++;
-                // Acknowledge next_sync
-                ack_one_sync(next_sync);
-            }
-        }
-        return 1;
-    }
-    return 0;
-}
-
-void blockstore_impl_t::ack_one_sync(blockstore_op_t *op)
-{
-    // Handle states
-    for (auto it = PRIV(op)->sync_big_writes.begin(); it != PRIV(op)->sync_big_writes.end(); it++)
-    {
-#ifdef BLOCKSTORE_DEBUG
-        printf("Ack sync big %lu:%lu v%lu\n", it->oid.inode, it->oid.stripe, it->version);
-#endif
-        auto & unstab = unstable_writes[it->oid];
-        unstab = unstab < it->version ? it->version : unstab;
-        dirty_db[*it].state = ST_D_META_SYNCED;
-    }
-    for (auto it = PRIV(op)->sync_small_writes.begin(); it != PRIV(op)->sync_small_writes.end(); it++)
-    {
-#ifdef BLOCKSTORE_DEBUG
-        printf("Ack sync small %lu:%lu v%lu\n", it->oid.inode, it->oid.stripe, it->version);
-#endif
-        auto & unstab = unstable_writes[it->oid];
-        unstab = unstab < it->version ? it->version : unstab;
-        dirty_db[*it].state = dirty_db[*it].state == ST_DEL_WRITTEN ? ST_DEL_SYNCED : ST_J_SYNCED;
-    }
-    in_progress_syncs.erase(PRIV(op)->in_progress_ptr);
-    op->retval = 0;
-    FINISH_OP(op);
-}

blockstore_write.cpp

@@ -1,328 +0,0 @@
-#include "blockstore_impl.h"
-
-bool blockstore_impl_t::enqueue_write(blockstore_op_t *op)
-{
-    // Check or assign version number
-    bool found = false, deleted = false, is_del = (op->opcode == BS_OP_DELETE);
-    uint64_t version = 1;
-    if (dirty_db.size() > 0)
-    {
-        auto dirty_it = dirty_db.upper_bound((obj_ver_id){
-            .oid = op->oid,
-            .version = UINT64_MAX,
-        });
-        dirty_it--; // segfaults when dirty_db is empty
-        if (dirty_it != dirty_db.end() && dirty_it->first.oid == op->oid)
-        {
-            found = true;
-            version = dirty_it->first.version + 1;
-            deleted = IS_DELETE(dirty_it->second.state);
-        }
-    }
-    if (!found)
-    {
-        auto clean_it = clean_db.find(op->oid);
-        if (clean_it != clean_db.end())
-        {
-            version = clean_it->second.version + 1;
-        }
-        else
-        {
-            deleted = true;
-        }
-    }
-    if (op->version == 0)
-    {
-        op->version = version;
-    }
-    else if (op->version < version)
-    {
-        // Invalid version requested
-        op->retval = -EINVAL;
-        return false;
-    }
-    if (deleted && is_del)
-    {
-        // Already deleted
-        op->retval = 0;
-        return false;
-    }
-    // Immediately add the operation into dirty_db, so subsequent reads could see it
-#ifdef BLOCKSTORE_DEBUG
-    printf("%s %lu:%lu v%lu\n", is_del ? "Delete" : "Write", op->oid.inode, op->oid.stripe, op->version);
-#endif
-    dirty_db.emplace((obj_ver_id){
-        .oid = op->oid,
-        .version = op->version,
-    }, (dirty_entry){
-        .state = (uint32_t)(
-            is_del
-                ? ST_DEL_IN_FLIGHT
-                : (op->len == block_size || deleted ? ST_D_IN_FLIGHT : ST_J_IN_FLIGHT)
-        ),
-        .flags = 0,
-        .location = 0,
-        .offset = is_del ? 0 : op->offset,
-        .len = is_del ? 0 : op->len,
-        .journal_sector = 0,
-    });
-    return true;
-}
-
-// First step of the write algorithm: dequeue operation and submit initial write(s)
-int blockstore_impl_t::dequeue_write(blockstore_op_t *op)
-{
-    auto dirty_it = dirty_db.find((obj_ver_id){
-        .oid = op->oid,
-        .version = op->version,
-    });
-    if (dirty_it->second.state == ST_D_IN_FLIGHT)
-    {
-        blockstore_journal_check_t space_check(this);
-        if (!space_check.check_available(op, unsynced_big_writes.size() + 1, sizeof(journal_entry_big_write), JOURNAL_STABILIZE_RESERVATION))
-        {
-            return 0;
-        }
-        // Big (redirect) write
-        uint64_t loc = data_alloc->find_free();
-        if (loc == UINT64_MAX)
-        {
-            // no space
-            if (flusher->is_active())
-            {
-                // hope that some space will be available after flush
-                PRIV(op)->wait_for = WAIT_FREE;
-                return 0;
-            }
-            op->retval = -ENOSPC;
-            FINISH_OP(op);
-            return 1;
-        }
-        BS_SUBMIT_GET_SQE(sqe, data);
-        dirty_it->second.location = loc << block_order;
-        dirty_it->second.state = ST_D_SUBMITTED;
-#ifdef BLOCKSTORE_DEBUG
-        printf("Allocate block %lu\n", loc);
-#endif
-        data_alloc->set(loc, true);
-        uint64_t stripe_offset = (op->offset % bitmap_granularity);
-        uint64_t stripe_end = (op->offset + op->len) % bitmap_granularity;
-        // Zero fill up to bitmap_granularity
-        int vcnt = 0;
-        if (stripe_offset)
-        {
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, stripe_offset };
-        }
-        PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ op->buf, op->len };
-        if (stripe_end)
-        {
-            stripe_end = bitmap_granularity - stripe_end;
-            PRIV(op)->iov_zerofill[vcnt++] = (struct iovec){ zero_object, stripe_end };
-        }
-        data->iov.iov_len = op->len + stripe_offset + stripe_end; // to check it in the callback
-        data->callback = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        my_uring_prep_writev(
-            sqe, data_fd, PRIV(op)->iov_zerofill, vcnt, data_offset + (loc << block_order) + op->offset - stripe_offset
-        );
-        PRIV(op)->pending_ops = 1;
-        PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 0;
-        // Remember big write as unsynced
-        unsynced_big_writes.push_back((obj_ver_id){
-            .oid = op->oid,
-            .version = op->version,
-        });
-    }
-    else
-    {
-        // Small (journaled) write
-        // First check if the journal has sufficient space
-        blockstore_journal_check_t space_check(this);
-        if (unsynced_big_writes.size() && !space_check.check_available(op, unsynced_big_writes.size(), sizeof(journal_entry_big_write), 0)
-            || !space_check.check_available(op, 1, sizeof(journal_entry_small_write), op->len + JOURNAL_STABILIZE_RESERVATION))
-        {
-            return 0;
-        }
-        // There is sufficient space. Get SQE(s)
-        struct io_uring_sqe *sqe1 = NULL;
-        if ((journal_block_size - journal.in_sector_pos) < sizeof(journal_entry_small_write) &&
-            journal.sector_info[journal.cur_sector].dirty)
-        {
-            // Write current journal sector only if it's dirty and full
-            BS_SUBMIT_GET_SQE_DECL(sqe1);
-        }
-        struct io_uring_sqe *sqe2 = NULL;
-        if (op->len > 0)
-        {
-            BS_SUBMIT_GET_SQE_DECL(sqe2);
-        }
-        // Got SQEs. Prepare previous journal sector write if required
-        auto cb = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-        if (sqe1)
-        {
-            prepare_journal_sector_write(journal, journal.cur_sector, sqe1, cb);
-            // FIXME rename to min/max _flushing
-            PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-            PRIV(op)->pending_ops++;
-        }
-        else
-        {
-            PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 0;
-        }
-        // Then pre-fill journal entry
-        journal_entry_small_write *je = (journal_entry_small_write*)
-            prefill_single_journal_entry(journal, JE_SMALL_WRITE, sizeof(journal_entry_small_write));
-        dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
-        journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
-#ifdef BLOCKSTORE_DEBUG
-        printf("journal offset %lu is used by %lu:%lu v%lu\n", dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version);
-#endif
-        // Figure out where data will be
-        journal.next_free = (journal.next_free + op->len) <= journal.len ? journal.next_free : journal_block_size;
-        je->oid = op->oid;
-        je->version = op->version;
-        je->offset = op->offset;
-        je->len = op->len;
-        je->data_offset = journal.next_free;
-        je->crc32_data = crc32c(0, op->buf, op->len);
-        je->crc32 = je_crc32((journal_entry*)je);
-        journal.crc32_last = je->crc32;
-        if (op->len > 0)
-        {
-            // Prepare journal data write
-            if (journal.inmemory)
-            {
-                // Copy data
-                memcpy(journal.buffer + journal.next_free, op->buf, op->len);
-            }
-            ring_data_t *data2 = ((ring_data_t*)sqe2->user_data);
-            data2->iov = (struct iovec){ op->buf, op->len };
-            data2->callback = cb;
-            my_uring_prep_writev(
-                sqe2, journal.fd, &data2->iov, 1, journal.offset + journal.next_free
-            );
-            PRIV(op)->pending_ops++;
-        }
-        else
-        {
-            // Zero-length overwrite. Allowed to bump object version in EC placement groups without actually writing data
-        }
-        dirty_it->second.location = journal.next_free;
-        dirty_it->second.state = ST_J_SUBMITTED;
-        journal.next_free += op->len;
-        if (journal.next_free >= journal.len)
-        {
-            journal.next_free = journal_block_size;
-        }
-        // Remember small write as unsynced
-        unsynced_small_writes.push_back((obj_ver_id){
-            .oid = op->oid,
-            .version = op->version,
-        });
-        if (!PRIV(op)->pending_ops)
-        {
-            ack_write(op);
-        }
-    }
-    return 1;
-}
-
-void blockstore_impl_t::handle_write_event(ring_data_t *data, blockstore_op_t *op)
-{
-    live = true;
-    if (data->res != data->iov.iov_len)
-    {
-        // FIXME: our state becomes corrupted after a write error. maybe do something better than just die
-        throw std::runtime_error(
-            "write operation failed ("+std::to_string(data->res)+" != "+std::to_string(data->iov.iov_len)+
-            "). in-memory state is corrupted. AAAAAAAaaaaaaaaa!!!111"
-        );
-    }
-    PRIV(op)->pending_ops--;
-    if (PRIV(op)->pending_ops == 0)
-    {
-        release_journal_sectors(op);
-        ack_write(op);
-    }
-}
-
-void blockstore_impl_t::release_journal_sectors(blockstore_op_t *op)
-{
-    // Release used journal sectors
-    if (PRIV(op)->min_used_journal_sector > 0 &&
-        PRIV(op)->max_used_journal_sector > 0)
-    {
-        uint64_t s = PRIV(op)->min_used_journal_sector;
-        while (1)
-        {
-            journal.sector_info[s-1].usage_count--;
-            if (s == PRIV(op)->max_used_journal_sector)
-                break;
-            s = 1 + s % journal.sector_count;
-        }
-        PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 0;
-    }
-}
-
-void blockstore_impl_t::ack_write(blockstore_op_t *op)
-{
-    // Switch object state
-    auto & dirty_entry = dirty_db[(obj_ver_id){
-        .oid = op->oid,
-        .version = op->version,
-    }];
-#ifdef BLOCKSTORE_DEBUG
-    printf("Ack write %lu:%lu v%lu = %d\n", op->oid.inode, op->oid.stripe, op->version, dirty_entry.state);
-#endif
-    if (dirty_entry.state == ST_J_SUBMITTED)
-    {
-        dirty_entry.state = ST_J_WRITTEN;
-    }
-    else if (dirty_entry.state == ST_D_SUBMITTED)
-    {
-        dirty_entry.state = ST_D_WRITTEN;
-    }
-    else if (dirty_entry.state == ST_DEL_SUBMITTED)
-    {
-        dirty_entry.state = ST_DEL_WRITTEN;
-    }
-    // Acknowledge write without sync
-    op->retval = op->len;
-    FINISH_OP(op);
-}
-
-int blockstore_impl_t::dequeue_del(blockstore_op_t *op)
-{
-    auto dirty_it = dirty_db.find((obj_ver_id){
-        .oid = op->oid,
-        .version = op->version,
-    });
-    blockstore_journal_check_t space_check(this);
-    if (!space_check.check_available(op, 1, sizeof(journal_entry_del), 0))
-    {
-        return 0;
-    }
-    BS_SUBMIT_GET_ONLY_SQE(sqe);
-    // Prepare journal sector write
-    journal_entry_del *je = (journal_entry_del*)
-        prefill_single_journal_entry(journal, JE_DELETE, sizeof(struct journal_entry_del));
-    dirty_it->second.journal_sector = journal.sector_info[journal.cur_sector].offset;
-    journal.used_sectors[journal.sector_info[journal.cur_sector].offset]++;
-#ifdef BLOCKSTORE_DEBUG
-    printf("journal offset %lu is used by %lu:%lu v%lu\n", dirty_it->second.journal_sector, dirty_it->first.oid.inode, dirty_it->first.oid.stripe, dirty_it->first.version);
-#endif
-    je->oid = op->oid;
-    je->version = op->version;
-    je->crc32 = je_crc32((journal_entry*)je);
-    journal.crc32_last = je->crc32;
-    auto cb = [this, op](ring_data_t *data) { handle_write_event(data, op); };
-    prepare_journal_sector_write(journal, journal.cur_sector, sqe, cb);
-    PRIV(op)->min_used_journal_sector = PRIV(op)->max_used_journal_sector = 1 + journal.cur_sector;
-    PRIV(op)->pending_ops = 1;
-    dirty_it->second.state = ST_DEL_SUBMITTED;
-    // Remember small write as unsynced
-    unsynced_small_writes.push_back((obj_ver_id){
-        .oid = op->oid,
-        .version = op->version,
-    });
-    return 1;
-}

copy-fio-includes.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+gcc -I. -E -o fio_headers.i src/fio_headers.h
+
+rm -rf fio-copy
+for i in `grep -Po 'fio/[^"]+' fio_headers.i | sort | uniq`; do
+    j=${i##fio/}
+    p=$(dirname $j)
+    mkdir -p fio-copy/$p
+    cp $i fio-copy/$j
+done
+
+rm fio_headers.i

copy-qemu-includes.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+#cd qemu
+#debian/rules b/configure-stamp
+#cd b/qemu; make qapi
+
+gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
+    -I qemu/include -E -o qemu_driver.i src/qemu_driver.c
+
+rm -rf qemu-copy
+for i in `grep -Po 'qemu/[^"]+' qemu_driver.i | sort | uniq`; do
+    j=${i##qemu/}
+    p=$(dirname $j)
+    mkdir -p qemu-copy/$p
+    cp $i qemu-copy/$j
+done
+
+rm qemu_driver.i

cpp-btree

@@ -0,0 +1 @@
+Subproject commit 8de8b467acbca50cfd8835c20e0e379110f3b32b

csi/.dockerignore

@@ -0,0 +1,2 @@
+vitastor-csi
+Dockerfile

csi/Dockerfile

@@ -0,0 +1,48 @@
+# Compile stage
+FROM golang:bookworm AS build
+
+ADD go.sum go.mod /app/
+RUN cd /app; CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go mod download -x
+ADD . /app
+RUN perl -i -e '$/ = undef; while(<>) { s/\n\s*(\{\s*\n)/$1\n/g; s/\}(\s*\n\s*)else\b/$1} else/g; print; }' `find /app -name '*.go'` && \
+    cd /app && \
+    CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -o vitastor-csi
+
+# Final stage
+FROM debian:bookworm
+
+LABEL maintainers="Vitaliy Filippov <vitalif@yourcmc.ru>"
+LABEL description="Vitastor CSI Driver"
+
+ENV NODE_ID=""
+ENV CSI_ENDPOINT=""
+
+RUN apt-get update && \
+    apt-get install -y wget && \
+    (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
+    apt-get update && \
+    apt-get install -y e2fsprogs xfsprogs kmod iproute2 \
+        # dependencies of qemu-storage-daemon
+        libnuma1 liburing2 libglib2.0-0 libfuse3-3 libaio1 libzstd1 libnettle8 \
+        libgmp10 libhogweed6 libp11-kit0 libidn2-0 libunistring2 libtasn1-6 libpcre2-8-0 libffi8 && \
+    apt-get clean && \
+    (echo options nbd nbds_max=128 > /etc/modprobe.d/nbd.conf)
+
+COPY --from=build /app/vitastor-csi /bin/
+
+RUN (echo deb http://vitastor.io/debian bookworm main > /etc/apt/sources.list.d/vitastor.list) && \
+    ((echo 'Package: *'; echo 'Pin: origin "vitastor.io"'; echo 'Pin-Priority: 1000') > /etc/apt/preferences.d/vitastor.pref) && \
+    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg && \
+    apt-get update && \
+    apt-get install -y vitastor-client && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-8.1.2%2Bds-1%2Bvitastor1/qemu-utils_8.1.2%2Bds-1%2Bvitastor1_amd64.deb && \
+    wget https://vitastor.io/archive/qemu/qemu-bookworm-8.1.2%2Bds-1%2Bvitastor1/qemu-block-extra_8.1.2%2Bds-1%2Bvitastor1_amd64.deb && \
+    dpkg -x qemu-utils*.deb tmp1 && \
+    dpkg -x qemu-block-extra*.deb tmp1 && \
+    cp -a tmp1/usr/bin/qemu-storage-daemon /usr/bin/ && \
+    mkdir -p /usr/lib/x86_64-linux-gnu/qemu && \
+    cp -a tmp1/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/ && \
+    rm -rf tmp1 *.deb && \
+    apt-get clean
+
+ENTRYPOINT ["/bin/vitastor-csi"]

csi/Makefile

@@ -0,0 +1,9 @@
+VERSION ?= v1.4.4
+
+all: build push
+
+build:
+	@docker build --rm -t vitalif/vitastor-csi:$(VERSION) .
+
+push:
+	@docker push vitalif/vitastor-csi:$(VERSION)

csi/deploy/000-csi-namespace.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: vitastor-system

csi/deploy/001-csi-config-map.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: v1
+kind: ConfigMap
+data:
+  # You can add multiple configuration files here to use a multi-cluster setup
+  vitastor.conf: |-
+    {"etcd_address":"http://192.168.7.2:2379","etcd_prefix":"/vitastor"}
+metadata:
+  namespace: vitastor-system
+  name: vitastor-config

csi/deploy/002-csi-nodeplugin-rbac.yaml

@@ -0,0 +1,37 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get"]
+  # allow to read Vault Token and connection options from the Tenants namespace
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin
+subjects:
+  - kind: ServiceAccount
+    name: vitastor-csi-nodeplugin
+    namespace: vitastor-system
+roleRef:
+  kind: ClusterRole
+  name: vitastor-csi-nodeplugin
+  apiGroup: rbac.authorization.k8s.io

csi/deploy/003-csi-nodeplugin-psp.yaml

@@ -0,0 +1,72 @@
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin-psp
+spec:
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+    - 'SYS_ADMIN'
+  fsGroup:
+    rule: RunAsAny
+  privileged: true
+  hostNetwork: true
+  hostPID: true
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: '/dev'
+      readOnly: false
+    - pathPrefix: '/run/mount'
+      readOnly: false
+    - pathPrefix: '/sys'
+      readOnly: false
+    - pathPrefix: '/lib/modules'
+      readOnly: true
+    - pathPrefix: '/var/lib/kubelet/pods'
+      readOnly: false
+    - pathPrefix: '/var/lib/kubelet/plugins/csi.vitastor.io'
+      readOnly: false
+    - pathPrefix: '/var/lib/kubelet/plugins_registry'
+      readOnly: false
+    - pathPrefix: '/var/lib/kubelet/plugins'
+      readOnly: false
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin-psp
+rules:
+  - apiGroups: ['policy']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['vitastor-csi-nodeplugin-psp']
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-nodeplugin-psp
+subjects:
+  - kind: ServiceAccount
+    name: vitastor-csi-nodeplugin
+    namespace: vitastor-system
+roleRef:
+  kind: Role
+  name: vitastor-csi-nodeplugin-psp
+  apiGroup: rbac.authorization.k8s.io

csi/deploy/004-csi-nodeplugin.yaml

@@ -0,0 +1,145 @@
+---
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  namespace: vitastor-system
+  name: csi-vitastor
+spec:
+  selector:
+    matchLabels:
+      app: csi-vitastor
+  template:
+    metadata:
+      namespace: vitastor-system
+      labels:
+        app: csi-vitastor
+    spec:
+      serviceAccountName: vitastor-csi-nodeplugin
+      hostNetwork: true
+      hostPID: true
+      priorityClassName: system-node-critical
+      # to use e.g. Rook orchestrated cluster, and mons' FQDN is
+      # resolved through k8s service, set dns policy to cluster first
+      dnsPolicy: ClusterFirstWithHostNet
+      containers:
+        - name: driver-registrar
+          # This is necessary only for systems with SELinux, where
+          # non-privileged sidecar containers cannot access unix domain socket
+          # created by privileged CSI driver container.
+          securityContext:
+            privileged: true
+          image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.2.0
+          args:
+            - "--v=5"
+            - "--csi-address=/csi/csi.sock"
+            - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.vitastor.io/csi.sock"
+          env:
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - name: registration-dir
+              mountPath: /registration
+        - name: csi-vitastor
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+            allowPrivilegeEscalation: true
+          image: vitalif/vitastor-csi:v1.4.4
+          args:
+            - "--node=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          imagePullPolicy: "IfNotPresent"
+          ports:
+          - containerPort: 9898
+            name: healthz
+            protocol: TCP
+          livenessProbe:
+            failureThreshold: 5
+            httpGet:
+              path: /healthz
+              port: healthz
+            initialDelaySeconds: 10
+            timeoutSeconds: 3
+            periodSeconds: 2
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /run/mount
+              name: host-mount
+            - mountPath: /run/vitastor-csi
+              name: run-vitastor-csi
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: vitastor-config
+              mountPath: /etc/vitastor
+            - name: plugin-dir
+              mountPath: /var/lib/kubelet/plugins
+              mountPropagation: "Bidirectional"
+            - name: mountpoint-dir
+              mountPath: /var/lib/kubelet/pods
+              mountPropagation: "Bidirectional"
+        - name: liveness-probe
+          securityContext:
+            privileged: true
+          image: quay.io/k8scsi/livenessprobe:v1.1.0
+          args:
+            - "--csi-address=$(CSI_ENDPOINT)"
+            - "--health-port=9898"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi.sock
+          volumeMounts:
+          - mountPath: /csi
+            name: socket-dir
+      volumes:
+        - name: socket-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins/csi.vitastor.io
+            type: DirectoryOrCreate
+        - name: plugin-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins
+            type: Directory
+        - name: mountpoint-dir
+          hostPath:
+            path: /var/lib/kubelet/pods
+            type: DirectoryOrCreate
+        - name: registration-dir
+          hostPath:
+            path: /var/lib/kubelet/plugins_registry/
+            type: Directory
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: host-mount
+          hostPath:
+            path: /run/mount
+        - name: run-vitastor-csi
+          hostPath:
+            path: /run/vitastor-csi
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: vitastor-config
+          configMap:
+            name: vitastor-config

csi/deploy/005-csi-provisioner-rbac.yaml

@@ -0,0 +1,105 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-provisioner
+
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-external-provisioner-runner
+rules:
+  - apiGroups: [""]
+    resources: ["nodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["events"]
+    verbs: ["list", "watch", "create", "update", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumes"]
+    verbs: ["get", "list", "watch", "create", "update", "delete", "patch"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims"]
+    verbs: ["get", "list", "watch", "update"]
+  - apiGroups: [""]
+    resources: ["persistentvolumeclaims/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["storageclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots"]
+    verbs: ["get", "list", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshots/status"]
+    verbs: ["get", "list", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents"]
+    verbs: ["create", "get", "list", "watch", "update", "delete", "patch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotclasses"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments"]
+    verbs: ["get", "list", "watch", "update", "patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["volumeattachments/status"]
+    verbs: ["patch"]
+  - apiGroups: ["storage.k8s.io"]
+    resources: ["csinodes"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: ["snapshot.storage.k8s.io"]
+    resources: ["volumesnapshotcontents/status"]
+    verbs: ["update", "patch"]
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get"]
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-provisioner-role
+subjects:
+  - kind: ServiceAccount
+    name: vitastor-csi-provisioner
+    namespace: vitastor-system
+roleRef:
+  kind: ClusterRole
+  name: vitastor-external-provisioner-runner
+  apiGroup: rbac.authorization.k8s.io
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-external-provisioner-cfg
+rules:
+  - apiGroups: [""]
+    resources: ["configmaps"]
+    verbs: ["get", "list", "watch", "create", "update", "delete"]
+  - apiGroups: ["coordination.k8s.io"]
+    resources: ["leases"]
+    verbs: ["get", "watch", "list", "delete", "update", "create"]
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vitastor-csi-provisioner-role-cfg
+  namespace: vitastor-system
+subjects:
+  - kind: ServiceAccount
+    name: vitastor-csi-provisioner
+    namespace: vitastor-system
+roleRef:
+  kind: Role
+  name: vitastor-external-provisioner-cfg
+  apiGroup: rbac.authorization.k8s.io

csi/deploy/006-csi-provisioner-psp.yaml

@@ -0,0 +1,60 @@
+---
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-provisioner-psp
+spec:
+  allowPrivilegeEscalation: true
+  allowedCapabilities:
+    - 'SYS_ADMIN'
+  fsGroup:
+    rule: RunAsAny
+  privileged: true
+  runAsUser:
+    rule: RunAsAny
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: RunAsAny
+  volumes:
+    - 'configMap'
+    - 'emptyDir'
+    - 'projected'
+    - 'secret'
+    - 'downwardAPI'
+    - 'hostPath'
+  allowedHostPaths:
+    - pathPrefix: '/dev'
+      readOnly: false
+    - pathPrefix: '/sys'
+      readOnly: false
+    - pathPrefix: '/lib/modules'
+      readOnly: true
+
+---
+kind: Role
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  namespace: vitastor-system
+  name: vitastor-csi-provisioner-psp
+rules:
+  - apiGroups: ['policy']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames: ['vitastor-csi-provisioner-psp']
+
+---
+kind: RoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: vitastor-csi-provisioner-psp
+  namespace: vitastor-system
+subjects:
+  - kind: ServiceAccount
+    name: vitastor-csi-provisioner
+    namespace: vitastor-system
+roleRef:
+  kind: Role
+  name: vitastor-csi-provisioner-psp
+  apiGroup: rbac.authorization.k8s.io

csi/deploy/007-csi-provisioner.yaml

@@ -0,0 +1,164 @@
+---
+kind: Service
+apiVersion: v1
+metadata:
+  namespace: vitastor-system
+  name: csi-vitastor-provisioner
+  labels:
+    app: csi-metrics
+spec:
+  selector:
+    app: csi-vitastor-provisioner
+  ports:
+    - name: http-metrics
+      port: 8080
+      protocol: TCP
+      targetPort: 8680
+
+---
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  namespace: vitastor-system
+  name: csi-vitastor-provisioner
+spec:
+  replicas: 3
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 1
+      maxSurge: 0
+  selector:
+    matchLabels:
+      app: csi-vitastor-provisioner
+  template:
+    metadata:
+      namespace: vitastor-system
+      labels:
+        app: csi-vitastor-provisioner
+    spec:
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: app
+                    operator: In
+                    values:
+                      - csi-vitastor-provisioner
+              topologyKey: "kubernetes.io/hostname"
+      serviceAccountName: vitastor-csi-provisioner
+      priorityClassName: system-cluster-critical
+      containers:
+        - name: csi-provisioner
+          image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--retry-interval-start=500ms"
+            - "--leader-election=true"
+            #  set it to true to use topology based provisioning
+            - "--feature-gates=Topology=false"
+            # if fstype is not specified in storageclass, ext4 is default
+            - "--default-fstype=ext4"
+            - "--extra-create-metadata=true"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-snapshotter
+          image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.0.0
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--leader-election=true"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          securityContext:
+            privileged: true
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-attacher
+          image: k8s.gcr.io/sig-storage/csi-attacher:v3.1.0
+          args:
+            - "--v=5"
+            - "--csi-address=$(ADDRESS)"
+            - "--leader-election=true"
+            - "--retry-interval-start=500ms"
+          env:
+            - name: ADDRESS
+              value: /csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-resizer
+          image: k8s.gcr.io/sig-storage/csi-resizer:v1.1.0
+          args:
+            - "--csi-address=$(ADDRESS)"
+            - "--v=5"
+            - "--timeout=150s"
+            - "--leader-election"
+            - "--retry-interval-start=500ms"
+            - "--handle-volume-inuse-error=false"
+          env:
+            - name: ADDRESS
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+        - name: csi-vitastor
+          securityContext:
+            privileged: true
+            capabilities:
+              add: ["SYS_ADMIN"]
+          image: vitalif/vitastor-csi:v1.4.4
+          args:
+            - "--node=$(NODE_ID)"
+            - "--endpoint=$(CSI_ENDPOINT)"
+          env:
+            - name: NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+            - name: CSI_ENDPOINT
+              value: unix:///csi/csi-provisioner.sock
+          imagePullPolicy: "IfNotPresent"
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /csi
+            - mountPath: /dev
+              name: host-dev
+            - mountPath: /sys
+              name: host-sys
+            - mountPath: /lib/modules
+              name: lib-modules
+              readOnly: true
+            - name: vitastor-config
+              mountPath: /etc/vitastor
+      volumes:
+        - name: host-dev
+          hostPath:
+            path: /dev
+        - name: host-sys
+          hostPath:
+            path: /sys
+        - name: lib-modules
+          hostPath:
+            path: /lib/modules
+        - name: socket-dir
+          emptyDir: {
+            medium: "Memory"
+          }
+        - name: vitastor-config
+          configMap:
+            name: vitastor-config

csi/deploy/008-csi-driver.yaml

@@ -0,0 +1,11 @@
+---
+# if Kubernetes version is less than 1.18 change
+# apiVersion to storage.k8s.io/v1betav1
+apiVersion: storage.k8s.io/v1
+kind: CSIDriver
+metadata:
+  namespace: vitastor-system
+  name: csi.vitastor.io
+spec:
+  attachRequired: true
+  podInfoOnMount: false

csi/deploy/009-storage-class.yaml

@@ -0,0 +1,17 @@
+---
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  namespace: vitastor-system
+  name: vitastor
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+provisioner: csi.vitastor.io
+volumeBindingMode: Immediate
+parameters:
+  etcdVolumePrefix: ""
+  poolId: "1"
+  # you can choose other configuration file if you have it in the config map
+  # different etcd URLs and prefixes should also be put in the config
+  #configPath: "/etc/vitastor/vitastor.conf"
+allowVolumeExpansion: true

csi/deploy/example-pvc-block.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-vitastor-pvc-block
+spec:
+  storageClassName: vitastor
+  volumeMode: Block
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Gi

csi/deploy/example-pvc.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-vitastor-pvc
+spec:
+  storageClassName: vitastor
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi

csi/deploy/example-snapshot-class.yaml

@@ -0,0 +1,7 @@
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshotClass
+metadata:
+  name: vitastor-snapclass
+driver: csi.vitastor.io
+deletionPolicy: Delete
+parameters:

csi/deploy/example-snapshot-clone.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: test-vitastor-clone
+spec:
+  storageClassName: vitastor
+  dataSource:
+    name: snap1
+    kind: VolumeSnapshot
+    apiGroup: snapshot.storage.k8s.io
+  accessModes:
+    - ReadWriteOnce
+  resources:
+    requests:
+      storage: 10Gi

csi/deploy/example-snapshot.yaml

@@ -0,0 +1,8 @@
+apiVersion: snapshot.storage.k8s.io/v1
+kind: VolumeSnapshot
+metadata:
+  name: snap1
+spec:
+  volumeSnapshotClassName: vitastor-snapclass
+  source:
+    persistentVolumeClaimName: test-vitastor-pvc

csi/deploy/example-test-pod-block.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vitastor-test-block-pvc
+  namespace: default
+spec:
+  containers:
+  - name: vitastor-test-block-pvc
+    image: nginx
+    volumeDevices:
+      - name: data
+        devicePath: /dev/xvda
+  volumes:
+  - name: data
+    persistentVolumeClaim:
+      claimName: test-vitastor-pvc-block
+      readOnly: false

csi/deploy/example-test-pod.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: vitastor-test-nginx
+  namespace: default
+spec:
+  containers:
+   - name: vitastor-test-nginx
+     image: nginx
+     volumeMounts:
+       - mountPath: /usr/share/nginx/html/s3
+         name: data
+  volumes:
+   - name: data
+     persistentVolumeClaim:
+       claimName: test-vitastor-pvc
+       readOnly: false

csi/go.mod

@@ -0,0 +1,21 @@
+module vitastor.io/csi
+
+go 1.15
+
+require (
+	github.com/container-storage-interface/spec v1.4.0
+	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b
+	github.com/kubernetes-csi/csi-lib-utils v0.9.1
+	golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/grpc v1.33.1
+	google.golang.org/protobuf v1.24.0
+	k8s.io/klog v1.0.0
+	k8s.io/utils v0.0.0-20210305010621-2afb4311ab10
+)
+
+replace github.com/coreos/bbolt => go.etcd.io/bbolt v1.3.5
+
+replace go.etcd.io/bbolt => github.com/coreos/bbolt v1.3.5
+
+replace google.golang.org/grpc => google.golang.org/grpc v1.25.1

csi/go.sum

@@ -0,0 +1,366 @@
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI=
+github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630=
+github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0=
+github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q=
+github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA=
+github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g=
+github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0=
+github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM=
+github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc=
+github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/container-storage-interface/spec v1.2.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/container-storage-interface/spec v1.4.0 h1:ozAshSKxpJnYUfmkpZCTYyF/4MYeYlhdXbAvPvfGmkg=
+github.com/container-storage-interface/spec v1.4.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
+github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb h1:eBmm0M9fYhWpKZLjQUUKka/LtIxf46G4fxeEz5KJr9U=
+golang.org/x/net v0.0.0-20201202161906-c7110b5ffcbb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.25.1 h1:wdKvqQk7IttEw92GoRyKG2IDrUIpgpj6H6m81yfeMW0=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0 h1:UhZDfRO8JRQru4/+LlLE0BRKGF8L+PICnvYZmx/fEGA=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.19.0/go.mod h1:I1K45XlvTrDjmj5LoM5LuP/KYrhWbjUKT/SoPG0qTjw=
+k8s.io/apimachinery v0.19.0/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
+k8s.io/client-go v0.19.0/go.mod h1:H9E/VT95blcFQnlyShFgnFT9ZnJOAceiUHM3MlRC+mU=
+k8s.io/component-base v0.19.0/go.mod h1:dKsY8BxkA+9dZIAh2aWJLL/UdASFDNtGYTCItL4LM7Y=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0 h1:XRvcwJozkgZ1UQJmfMGpvRthQHOvihEhYtDfAaxMz/A=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
+k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+k8s.io/utils v0.0.0-20210305010621-2afb4311ab10 h1:u5rPykqiCpL+LBfjRkXvnK71gOgIdmq3eHUEkPrbeTI=
+k8s.io/utils v0.0.0-20210305010621-2afb4311ab10/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

csi/src/config.go

@@ -0,0 +1,22 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package vitastor
+
+const (
+    vitastorCSIDriverName    = "csi.vitastor.io"
+    vitastorCSIDriverVersion = "1.4.4"
+)
+
+// Config struct fills the parameters of request or user input
+type Config struct
+{
+    Endpoint string
+    NodeID   string
+}
+
+// NewConfig returns config struct to initialize new driver
+func NewConfig() *Config
+{
+    return &Config{}
+}

csi/src/controllerserver.go

@@ -0,0 +1,605 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package vitastor
+
+import (
+    "context"
+    "encoding/json"
+    "fmt"
+    "strings"
+    "bytes"
+    "strconv"
+    "time"
+    "os"
+    "os/exec"
+    "io/ioutil"
+
+    "github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+    "k8s.io/klog"
+
+    "google.golang.org/grpc/codes"
+    "google.golang.org/grpc/status"
+    "google.golang.org/protobuf/types/known/timestamppb"
+
+    "github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+const (
+    KB int64 = 1024
+    MB int64 = 1024 * KB
+    GB int64 = 1024 * MB
+    TB int64 = 1024 * GB
+    ETCD_TIMEOUT time.Duration = 15*time.Second
+)
+
+type InodeIndex struct
+{
+    Id uint64 `json:"id"`
+    PoolId uint64 `json:"pool_id"`
+}
+
+type InodeConfig struct
+{
+    Name string `json:"name"`
+    Size uint64 `json:"size,omitempty"`
+    ParentPool uint64 `json:"parent_pool,omitempty"`
+    ParentId uint64 `json:"parent_id,omitempty"`
+    Readonly bool `json:"readonly,omitempty"`
+    CreateTs uint64 `json:"create_ts,omitempty"`
+}
+
+type ControllerServer struct
+{
+    *Driver
+}
+
+// NewControllerServer create new instance controller
+func NewControllerServer(driver *Driver) *ControllerServer
+{
+    return &ControllerServer{
+        Driver: driver,
+    }
+}
+
+func GetConnectionParams(params map[string]string) (map[string]string, error)
+{
+    ctxVars := make(map[string]string)
+    configPath := params["configPath"]
+    if (configPath == "")
+    {
+        configPath = "/etc/vitastor/vitastor.conf"
+    }
+    else
+    {
+        ctxVars["configPath"] = configPath
+    }
+    config := make(map[string]interface{})
+    configFD, err := os.Open(configPath)
+    if (err != nil)
+    {
+        return nil, err
+    }
+    defer configFD.Close()
+    data, _ := ioutil.ReadAll(configFD)
+    json.Unmarshal(data, &config)
+    // Check etcd URL in the config, but do not use the explicit etcdUrl
+    // parameter for CLI calls, otherwise users won't be able to later
+    // change them - storage class parameters are saved in volume IDs
+    var etcdUrl []string
+    switch config["etcd_address"].(type)
+    {
+    case string:
+        url := strings.TrimSpace(config["etcd_address"].(string))
+        if (url != "")
+        {
+            etcdUrl = strings.Split(url, ",")
+        }
+    case []string:
+        etcdUrl = config["etcd_address"].([]string)
+    case []interface{}:
+        for _, url := range config["etcd_address"].([]interface{})
+        {
+            s, ok := url.(string)
+            if (ok)
+            {
+                etcdUrl = append(etcdUrl, s)
+            }
+        }
+    }
+    if (len(etcdUrl) == 0)
+    {
+        return nil, status.Error(codes.InvalidArgument, "etcd_address is missing in "+configPath)
+    }
+    return ctxVars, nil
+}
+
+func system(program string, args ...string) ([]byte, []byte, error)
+{
+    klog.Infof("Running "+program+" "+strings.Join(args, " "))
+    c := exec.Command(program, args...)
+    var stdout, stderr bytes.Buffer
+    c.Stdout, c.Stderr = &stdout, &stderr
+    err := c.Run()
+    if (err != nil)
+    {
+        stdoutStr, stderrStr := string(stdout.Bytes()), string(stderr.Bytes())
+        klog.Errorf(program+" "+strings.Join(args, " ")+" failed: %s, status %s\n", stdoutStr+stderrStr, err)
+        return nil, nil, status.Error(codes.Internal, stdoutStr+stderrStr+" (status "+err.Error()+")")
+    }
+    return stdout.Bytes(), stderr.Bytes(), nil
+}
+
+func invokeCLI(ctxVars map[string]string, args []string) ([]byte, error)
+{
+    if (ctxVars["configPath"] != "")
+    {
+        args = append(args, "--config_path", ctxVars["configPath"])
+    }
+    stdout, _, err := system("/usr/bin/vitastor-cli", args...)
+    return stdout, err
+}
+
+// Create the volume
+func (cs *ControllerServer) CreateVolume(ctx context.Context, req *csi.CreateVolumeRequest) (*csi.CreateVolumeResponse, error)
+{
+    klog.Infof("received controller create volume request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Errorf(codes.InvalidArgument, "request cannot be empty")
+    }
+    if (req.GetName() == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "name is a required field")
+    }
+    volumeCapabilities := req.GetVolumeCapabilities()
+    if (volumeCapabilities == nil)
+    {
+        return nil, status.Error(codes.InvalidArgument, "volume capabilities is a required field")
+    }
+
+    etcdVolumePrefix := req.Parameters["etcdVolumePrefix"]
+    poolId, _ := strconv.ParseUint(req.Parameters["poolId"], 10, 64)
+    if (poolId == 0)
+    {
+        return nil, status.Error(codes.InvalidArgument, "poolId is missing in storage class configuration")
+    }
+
+    volName := etcdVolumePrefix + req.GetName()
+    volSize := 1 * GB
+    if capRange := req.GetCapacityRange(); capRange != nil
+    {
+        volSize = ((capRange.GetRequiredBytes() + MB - 1) / MB) * MB
+    }
+
+    ctxVars, err := GetConnectionParams(req.Parameters)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    args := []string{ "create", volName, "-s", fmt.Sprintf("%v", volSize), "--pool", fmt.Sprintf("%v", poolId) }
+
+    // Support creation from snapshot
+    var src *csi.VolumeContentSource
+    if (req.VolumeContentSource.GetSnapshot() != nil)
+    {
+        snapId := req.VolumeContentSource.GetSnapshot().GetSnapshotId()
+        if (snapId != "")
+        {
+            snapVars := make(map[string]string)
+            err := json.Unmarshal([]byte(snapId), &snapVars)
+            if (err != nil)
+            {
+                return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+            }
+            args = append(args, "--parent", snapVars["name"]+"@"+snapVars["snapshot"])
+            src = &csi.VolumeContentSource{
+                Type: &csi.VolumeContentSource_Snapshot{
+                    Snapshot: &csi.VolumeContentSource_SnapshotSource{
+                        SnapshotId: snapId,
+                    },
+                },
+            }
+        }
+    }
+
+    // Create image using vitastor-cli
+    _, err = invokeCLI(ctxVars, args)
+    if (err != nil)
+    {
+        if (strings.Index(err.Error(), "already exists") > 0)
+        {
+            inodeCfg, err := invokeList(ctxVars, volName, true)
+            if (err != nil)
+            {
+                return nil, err
+            }
+            if (inodeCfg[0].Size < uint64(volSize))
+            {
+                return nil, status.Error(codes.Internal, "image "+volName+" is already created, but size is less than expected")
+            }
+        }
+        else
+        {
+            return nil, err
+        }
+    }
+
+    ctxVars["name"] = volName
+    volumeIdJson, _ := json.Marshal(ctxVars)
+    return &csi.CreateVolumeResponse{
+        Volume: &csi.Volume{
+            // Ugly, but VolumeContext isn't passed to DeleteVolume :-(
+            VolumeId: string(volumeIdJson),
+            CapacityBytes: volSize,
+            ContentSource: src,
+        },
+    }, nil
+}
+
+// DeleteVolume deletes the given volume
+func (cs *ControllerServer) DeleteVolume(ctx context.Context, req *csi.DeleteVolumeRequest) (*csi.DeleteVolumeResponse, error)
+{
+    klog.Infof("received controller delete volume request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Error(codes.InvalidArgument, "request cannot be empty")
+    }
+
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(req.VolumeId), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    volName := volVars["name"]
+
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    _, err = invokeCLI(ctxVars, []string{ "rm", volName })
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    return &csi.DeleteVolumeResponse{}, nil
+}
+
+// ControllerPublishVolume return Unimplemented error
+func (cs *ControllerServer) ControllerPublishVolume(ctx context.Context, req *csi.ControllerPublishVolumeRequest) (*csi.ControllerPublishVolumeResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ControllerUnpublishVolume return Unimplemented error
+func (cs *ControllerServer) ControllerUnpublishVolume(ctx context.Context, req *csi.ControllerUnpublishVolumeRequest) (*csi.ControllerUnpublishVolumeResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ValidateVolumeCapabilities checks whether the volume capabilities requested are supported.
+func (cs *ControllerServer) ValidateVolumeCapabilities(ctx context.Context, req *csi.ValidateVolumeCapabilitiesRequest) (*csi.ValidateVolumeCapabilitiesResponse, error)
+{
+    klog.Infof("received controller validate volume capability request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Errorf(codes.InvalidArgument, "request is nil")
+    }
+    volumeID := req.GetVolumeId()
+    if (volumeID == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "volumeId is nil")
+    }
+    volumeCapabilities := req.GetVolumeCapabilities()
+    if (volumeCapabilities == nil)
+    {
+        return nil, status.Error(codes.InvalidArgument, "volumeCapabilities is nil")
+    }
+
+    var volumeCapabilityAccessModes []*csi.VolumeCapability_AccessMode
+    for _, mode := range []csi.VolumeCapability_AccessMode_Mode{
+        csi.VolumeCapability_AccessMode_SINGLE_NODE_WRITER,
+        csi.VolumeCapability_AccessMode_MULTI_NODE_MULTI_WRITER,
+    } {
+        volumeCapabilityAccessModes = append(volumeCapabilityAccessModes, &csi.VolumeCapability_AccessMode{Mode: mode})
+    }
+
+    capabilitySupport := false
+    for _, capability := range volumeCapabilities
+    {
+        for _, volumeCapabilityAccessMode := range volumeCapabilityAccessModes
+        {
+            if (volumeCapabilityAccessMode.Mode == capability.AccessMode.Mode)
+            {
+                capabilitySupport = true
+            }
+        }
+    }
+
+    if (!capabilitySupport)
+    {
+        return nil, status.Errorf(codes.NotFound, "%v not supported", req.GetVolumeCapabilities())
+    }
+
+    return &csi.ValidateVolumeCapabilitiesResponse{
+        Confirmed: &csi.ValidateVolumeCapabilitiesResponse_Confirmed{
+            VolumeCapabilities: req.VolumeCapabilities,
+        },
+    }, nil
+}
+
+// ListVolumes returns a list of volumes
+func (cs *ControllerServer) ListVolumes(ctx context.Context, req *csi.ListVolumesRequest) (*csi.ListVolumesResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// GetCapacity returns the capacity of the storage pool
+func (cs *ControllerServer) GetCapacity(ctx context.Context, req *csi.GetCapacityRequest) (*csi.GetCapacityResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// ControllerGetCapabilities returns the capabilities of the controller service.
+func (cs *ControllerServer) ControllerGetCapabilities(ctx context.Context, req *csi.ControllerGetCapabilitiesRequest) (*csi.ControllerGetCapabilitiesResponse, error)
+{
+    functionControllerServerCapabilities := func(cap csi.ControllerServiceCapability_RPC_Type) *csi.ControllerServiceCapability
+    {
+        return &csi.ControllerServiceCapability{
+            Type: &csi.ControllerServiceCapability_Rpc{
+                Rpc: &csi.ControllerServiceCapability_RPC{
+                    Type: cap,
+                },
+            },
+        }
+    }
+
+    var controllerServerCapabilities []*csi.ControllerServiceCapability
+    for _, capability := range []csi.ControllerServiceCapability_RPC_Type{
+        csi.ControllerServiceCapability_RPC_CREATE_DELETE_VOLUME,
+        csi.ControllerServiceCapability_RPC_LIST_VOLUMES,
+        csi.ControllerServiceCapability_RPC_EXPAND_VOLUME,
+        csi.ControllerServiceCapability_RPC_CREATE_DELETE_SNAPSHOT,
+        csi.ControllerServiceCapability_RPC_LIST_SNAPSHOTS,
+        // TODO: csi.ControllerServiceCapability_RPC_CLONE_VOLUME,
+    } {
+        controllerServerCapabilities = append(controllerServerCapabilities, functionControllerServerCapabilities(capability))
+    }
+
+    return &csi.ControllerGetCapabilitiesResponse{
+        Capabilities: controllerServerCapabilities,
+    }, nil
+}
+
+func invokeList(ctxVars map[string]string, pattern string, expectExist bool) ([]InodeConfig, error)
+{
+    stat, err := invokeCLI(ctxVars, []string{ "ls", "--json", pattern })
+    if (err != nil)
+    {
+        return nil, err
+    }
+    var inodeCfg []InodeConfig
+    err = json.Unmarshal(stat, &inodeCfg)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "Invalid JSON in vitastor-cli ls: "+err.Error())
+    }
+    if (expectExist && len(inodeCfg) == 0)
+    {
+        return nil, status.Error(codes.Internal, "Can't find expected image "+pattern+" via vitastor-cli ls")
+    }
+    return inodeCfg, nil
+}
+
+// CreateSnapshot create snapshot of an existing PV
+func (cs *ControllerServer) CreateSnapshot(ctx context.Context, req *csi.CreateSnapshotRequest) (*csi.CreateSnapshotResponse, error)
+{
+    klog.Infof("received controller create snapshot request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Errorf(codes.InvalidArgument, "request cannot be empty")
+    }
+    if (req.SourceVolumeId == "" || req.Name == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "source volume ID and snapshot name are required fields")
+    }
+
+    // snapshot name
+    snapName := req.Name
+
+    // req.VolumeId is an ugly json string in our case :)
+    ctxVars := make(map[string]string)
+    err := json.Unmarshal([]byte(req.SourceVolumeId), &ctxVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    volName := ctxVars["name"]
+
+    // Create image using vitastor-cli
+    _, err = invokeCLI(ctxVars, []string{ "create", "--snapshot", snapName, volName })
+    if (err != nil && strings.Index(err.Error(), "already exists") <= 0)
+    {
+        return nil, err
+    }
+
+    // Check created snapshot
+    inodeCfg, err := invokeList(ctxVars, volName+"@"+snapName, true)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    // Use ugly JSON snapshot ID again, DeleteSnapshot doesn't have context :-(
+    ctxVars["snapshot"] = snapName
+    snapIdJson, _ := json.Marshal(ctxVars)
+    return &csi.CreateSnapshotResponse{
+        Snapshot: &csi.Snapshot{
+            SizeBytes: int64(inodeCfg[0].Size),
+            SnapshotId: string(snapIdJson),
+            SourceVolumeId: req.SourceVolumeId,
+            CreationTime: &timestamppb.Timestamp{ Seconds: int64(inodeCfg[0].CreateTs) },
+            ReadyToUse: true,
+        },
+    }, nil
+}
+
+// DeleteSnapshot delete provided snapshot of a PV
+func (cs *ControllerServer) DeleteSnapshot(ctx context.Context, req *csi.DeleteSnapshotRequest) (*csi.DeleteSnapshotResponse, error)
+{
+    klog.Infof("received controller delete snapshot request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Errorf(codes.InvalidArgument, "request cannot be empty")
+    }
+    if (req.SnapshotId == "")
+    {
+        return nil, status.Error(codes.InvalidArgument, "snapshot ID is a required field")
+    }
+
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(req.SnapshotId), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "snapshot ID not in JSON format")
+    }
+    volName := volVars["name"]
+    snapName := volVars["snapshot"]
+
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    _, err = invokeCLI(ctxVars, []string{ "rm", volName+"@"+snapName })
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    return &csi.DeleteSnapshotResponse{}, nil
+}
+
+// ListSnapshots list the snapshots of a PV
+func (cs *ControllerServer) ListSnapshots(ctx context.Context, req *csi.ListSnapshotsRequest) (*csi.ListSnapshotsResponse, error)
+{
+    klog.Infof("received controller list snapshots request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Error(codes.InvalidArgument, "request cannot be empty")
+    }
+
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(req.SourceVolumeId), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    volName := volVars["name"]
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    inodeCfg, err := invokeList(ctxVars, volName+"@*", false)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    resp := &csi.ListSnapshotsResponse{}
+    for _, ino := range inodeCfg
+    {
+        snapName := ino.Name[len(volName)+1:]
+        if (len(req.StartingToken) > 0 && snapName < req.StartingToken)
+        {
+        }
+        else if (req.MaxEntries == 0 || len(resp.Entries) < int(req.MaxEntries))
+        {
+            volVars["snapshot"] = snapName
+            snapIdJson, _ := json.Marshal(volVars)
+            resp.Entries = append(resp.Entries, &csi.ListSnapshotsResponse_Entry{
+                Snapshot: &csi.Snapshot{
+                    SizeBytes: int64(ino.Size),
+                    SnapshotId: string(snapIdJson),
+                    SourceVolumeId: req.SourceVolumeId,
+                    CreationTime: &timestamppb.Timestamp{ Seconds: int64(ino.CreateTs) },
+                    ReadyToUse: true,
+                },
+            })
+        }
+        else
+        {
+            resp.NextToken = snapName
+            break
+        }
+    }
+
+    return resp, nil
+}
+
+// ControllerExpandVolume increases the size of a volume
+func (cs *ControllerServer) ControllerExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest) (*csi.ControllerExpandVolumeResponse, error)
+{
+    klog.Infof("received controller expand volume request %+v", protosanitizer.StripSecrets(req))
+    if (req == nil)
+    {
+        return nil, status.Error(codes.InvalidArgument, "request cannot be empty")
+    }
+    if (req.VolumeId == "" || req.CapacityRange == nil || req.CapacityRange.RequiredBytes == 0)
+    {
+        return nil, status.Error(codes.InvalidArgument, "VolumeId, CapacityRange and RequiredBytes are required fields")
+    }
+
+    volVars := make(map[string]string)
+    err := json.Unmarshal([]byte(req.VolumeId), &volVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    volName := volVars["name"]
+    ctxVars, err := GetConnectionParams(volVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    inodeCfg, err := invokeList(ctxVars, volName, true)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    if (req.CapacityRange.RequiredBytes > 0 && inodeCfg[0].Size < uint64(req.CapacityRange.RequiredBytes))
+    {
+        sz := ((req.CapacityRange.RequiredBytes+4095)/4096)*4096
+        _, err := invokeCLI(ctxVars, []string{ "modify", "--inc_size", "1", "--resize", fmt.Sprintf("%d", sz), volName })
+        if (err != nil)
+        {
+            return nil, err
+        }
+        inodeCfg, err = invokeList(ctxVars, volName, true)
+        if (err != nil)
+        {
+            return nil, err
+        }
+    }
+
+    return &csi.ControllerExpandVolumeResponse{
+        CapacityBytes: int64(inodeCfg[0].Size),
+        NodeExpansionRequired: false,
+    }, nil
+}
+
+// ControllerGetVolume get volume info
+func (cs *ControllerServer) ControllerGetVolume(ctx context.Context, req *csi.ControllerGetVolumeRequest) (*csi.ControllerGetVolumeResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}

csi/src/grpc.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package vitastor
+
+import (
+    "fmt"
+    "net"
+    "os"
+    "strings"
+    "sync"
+
+    "github.com/golang/glog"
+    "golang.org/x/net/context"
+    "google.golang.org/grpc"
+
+    "github.com/container-storage-interface/spec/lib/go/csi"
+    "github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+)
+
+// Defines Non blocking GRPC server interfaces
+type NonBlockingGRPCServer interface {
+    // Start services at the endpoint
+    Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer)
+    // Waits for the service to stop
+    Wait()
+    // Stops the service gracefully
+    Stop()
+    // Stops the service forcefully
+    ForceStop()
+}
+
+func NewNonBlockingGRPCServer() NonBlockingGRPCServer {
+    return &nonBlockingGRPCServer{}
+}
+
+// NonBlocking server
+type nonBlockingGRPCServer struct {
+    wg     sync.WaitGroup
+    server *grpc.Server
+}
+
+func (s *nonBlockingGRPCServer) Start(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
+
+    s.wg.Add(1)
+
+    go s.serve(endpoint, ids, cs, ns)
+
+    return
+}
+
+func (s *nonBlockingGRPCServer) Wait() {
+    s.wg.Wait()
+}
+
+func (s *nonBlockingGRPCServer) Stop() {
+    s.server.GracefulStop()
+}
+
+func (s *nonBlockingGRPCServer) ForceStop() {
+    s.server.Stop()
+}
+
+func (s *nonBlockingGRPCServer) serve(endpoint string, ids csi.IdentityServer, cs csi.ControllerServer, ns csi.NodeServer) {
+
+    proto, addr, err := ParseEndpoint(endpoint)
+    if err != nil {
+        glog.Fatal(err.Error())
+    }
+
+    if proto == "unix" {
+        addr = "/" + addr
+        if err := os.Remove(addr); err != nil && !os.IsNotExist(err) {
+            glog.Fatalf("Failed to remove %s, error: %s", addr, err.Error())
+        }
+    }
+
+    listener, err := net.Listen(proto, addr)
+    if err != nil {
+        glog.Fatalf("Failed to listen: %v", err)
+    }
+
+    opts := []grpc.ServerOption{
+        grpc.UnaryInterceptor(logGRPC),
+    }
+    server := grpc.NewServer(opts...)
+    s.server = server
+
+    if ids != nil {
+        csi.RegisterIdentityServer(server, ids)
+    }
+    if cs != nil {
+        csi.RegisterControllerServer(server, cs)
+    }
+    if ns != nil {
+        csi.RegisterNodeServer(server, ns)
+    }
+
+    glog.Infof("Listening for connections on address: %#v", listener.Addr())
+
+    server.Serve(listener)
+}
+
+func ParseEndpoint(ep string) (string, string, error) {
+    if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
+        s := strings.SplitN(ep, "://", 2)
+        if s[1] != "" {
+            return s[0], s[1], nil
+        }
+    }
+    return "", "", fmt.Errorf("Invalid endpoint: %v", ep)
+}
+
+func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+    glog.V(3).Infof("GRPC call: %s", info.FullMethod)
+    glog.V(5).Infof("GRPC request: %s", protosanitizer.StripSecrets(req))
+    resp, err := handler(ctx, req)
+    if err != nil {
+        glog.Errorf("GRPC error: %v", err)
+    } else {
+        glog.V(5).Infof("GRPC response: %s", protosanitizer.StripSecrets(resp))
+    }
+    return resp, err
+}

csi/src/identityserver.go

@@ -0,0 +1,67 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package vitastor
+
+import (
+    "context"
+
+    "github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+    "k8s.io/klog"
+
+    "github.com/container-storage-interface/spec/lib/go/csi"
+)
+
+// IdentityServer struct of Vitastor CSI driver with supported methods of CSI identity server spec.
+type IdentityServer struct
+{
+    *Driver
+}
+
+// NewIdentityServer create new instance identity
+func NewIdentityServer(driver *Driver) *IdentityServer
+{
+    return &IdentityServer{
+        Driver: driver,
+    }
+}
+
+// GetPluginInfo returns metadata of the plugin
+func (is *IdentityServer) GetPluginInfo(ctx context.Context, req *csi.GetPluginInfoRequest) (*csi.GetPluginInfoResponse, error)
+{
+    klog.Infof("received identity plugin info request %+v", protosanitizer.StripSecrets(req))
+    return &csi.GetPluginInfoResponse{
+        Name:          vitastorCSIDriverName,
+        VendorVersion: vitastorCSIDriverVersion,
+    }, nil
+}
+
+// GetPluginCapabilities returns available capabilities of the plugin
+func (is *IdentityServer) GetPluginCapabilities(ctx context.Context, req *csi.GetPluginCapabilitiesRequest) (*csi.GetPluginCapabilitiesResponse, error)
+{
+    klog.Infof("received identity plugin capabilities request %+v", protosanitizer.StripSecrets(req))
+    return &csi.GetPluginCapabilitiesResponse{
+        Capabilities: []*csi.PluginCapability{
+            {
+                Type: &csi.PluginCapability_Service_{
+                    Service: &csi.PluginCapability_Service{
+                        Type: csi.PluginCapability_Service_CONTROLLER_SERVICE,
+                    },
+                },
+            },
+            {
+                Type: &csi.PluginCapability_VolumeExpansion_{
+                    VolumeExpansion: &csi.PluginCapability_VolumeExpansion{
+                        Type: csi.PluginCapability_VolumeExpansion_OFFLINE,
+                    },
+                },
+            },
+        },
+    }, nil
+}
+
+// Probe returns the health and readiness of the plugin
+func (is *IdentityServer) Probe(ctx context.Context, req *csi.ProbeRequest) (*csi.ProbeResponse, error)
+{
+    return &csi.ProbeResponse{}, nil
+}

csi/src/nodeserver.go

@@ -0,0 +1,698 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package vitastor
+
+import (
+    "context"
+    "errors"
+    "encoding/json"
+    "fmt"
+    "os"
+    "os/exec"
+    "path/filepath"
+    "strconv"
+    "strings"
+    "syscall"
+    "time"
+
+    "google.golang.org/grpc/codes"
+    "google.golang.org/grpc/status"
+    "k8s.io/utils/mount"
+    utilexec "k8s.io/utils/exec"
+
+    "github.com/container-storage-interface/spec/lib/go/csi"
+    "github.com/kubernetes-csi/csi-lib-utils/protosanitizer"
+    "k8s.io/klog"
+)
+
+// NodeServer struct of Vitastor CSI driver with supported methods of CSI node server spec.
+type NodeServer struct
+{
+    *Driver
+    useVduse bool
+    stateDir string
+    mounter mount.Interface
+    restartInterval time.Duration
+}
+
+type DeviceState struct
+{
+    ConfigPath string `json:"configPath"`
+    VdpaId     string `json:"vdpaId"`
+    Image      string `json:"image"`
+    Blockdev   string `json:"blockdev"`
+    Readonly   bool   `json:"readonly"`
+    PidFile    string `json:"pidFile"`
+}
+
+// NewNodeServer create new instance node
+func NewNodeServer(driver *Driver) *NodeServer
+{
+    stateDir := os.Getenv("STATE_DIR")
+    if (stateDir == "")
+    {
+        stateDir = "/run/vitastor-csi"
+    }
+    if (stateDir[len(stateDir)-1] != '/')
+    {
+        stateDir += "/"
+    }
+    ns := &NodeServer{
+        Driver: driver,
+        useVduse: checkVduseSupport(),
+        stateDir: stateDir,
+        mounter: mount.New(""),
+    }
+    if (ns.useVduse)
+    {
+        ns.restoreVduseDaemons()
+        dur, err := time.ParseDuration(os.Getenv("RESTART_INTERVAL"))
+        if (err != nil)
+        {
+            dur = 10 * time.Second
+        }
+        ns.restartInterval = dur
+        if (ns.restartInterval != time.Duration(0))
+        {
+            go ns.restarter()
+        }
+    }
+    return ns
+}
+
+func checkVduseSupport() bool
+{
+    // Check VDUSE support (vdpa, vduse, virtio-vdpa kernel modules)
+    vduse := true
+    for _, mod := range []string{"vdpa", "vduse", "virtio-vdpa"}
+    {
+        _, err := os.Stat("/sys/module/"+mod)
+        if (err != nil)
+        {
+            if (!errors.Is(err, os.ErrNotExist))
+            {
+                klog.Errorf("failed to check /sys/module/%s: %v", mod, err)
+            }
+            c := exec.Command("/sbin/modprobe", mod)
+            c.Stdout = os.Stderr
+            c.Stderr = os.Stderr
+            err := c.Run()
+            if (err != nil)
+            {
+                klog.Errorf("/sbin/modprobe %s failed: %v", mod, err)
+                vduse = false
+                break
+            }
+        }
+    }
+    // Check that vdpa tool functions
+    if (vduse)
+    {
+        c := exec.Command("/sbin/vdpa", "-j", "dev")
+        c.Stderr = os.Stderr
+        err := c.Run()
+        if (err != nil)
+        {
+            klog.Errorf("/sbin/vdpa -j dev failed: %v", err)
+            vduse = false
+        }
+    }
+    if (!vduse)
+    {
+        klog.Errorf(
+            "Your host apparently has no VDUSE support. VDUSE support disabled, NBD will be used to map devices."+
+            " For VDUSE you need at least Linux 5.15 and the following kernel modules: vdpa, virtio-vdpa, vduse.",
+        )
+    }
+    return vduse
+}
+
+// NodeStageVolume mounts the volume to a staging path on the node.
+func (ns *NodeServer) NodeStageVolume(ctx context.Context, req *csi.NodeStageVolumeRequest) (*csi.NodeStageVolumeResponse, error)
+{
+    return &csi.NodeStageVolumeResponse{}, nil
+}
+
+// NodeUnstageVolume unstages the volume from the staging path
+func (ns *NodeServer) NodeUnstageVolume(ctx context.Context, req *csi.NodeUnstageVolumeRequest) (*csi.NodeUnstageVolumeResponse, error)
+{
+    return &csi.NodeUnstageVolumeResponse{}, nil
+}
+
+func Contains(list []string, s string) bool
+{
+    for i := 0; i < len(list); i++
+    {
+        if (list[i] == s)
+        {
+            return true
+        }
+    }
+    return false
+}
+
+func (ns *NodeServer) mapNbd(volName string, ctxVars map[string]string, readonly bool) (string, error)
+{
+    // Map NBD device
+    // FIXME: Check if already mapped
+    args := []string{
+        "map", "--image", volName,
+    }
+    if (ctxVars["configPath"] != "")
+    {
+        args = append(args, "--config_path", ctxVars["configPath"])
+    }
+    if (readonly)
+    {
+        args = append(args, "--readonly", "1")
+    }
+    stdout, stderr, err := system("/usr/bin/vitastor-nbd", args...)
+    dev := strings.TrimSpace(string(stdout))
+    if (dev == "")
+    {
+        return "", fmt.Errorf("vitastor-nbd did not return the name of NBD device. output: %s", stderr)
+    }
+    return dev, err
+}
+
+func (ns *NodeServer) unmapNbd(devicePath string)
+{
+    // unmap NBD device
+    unmapOut, unmapErr := exec.Command("/usr/bin/vitastor-nbd", "unmap", devicePath).CombinedOutput()
+    if (unmapErr != nil)
+    {
+        klog.Errorf("failed to unmap NBD device %s: %s, error: %v", devicePath, unmapOut, unmapErr)
+    }
+}
+
+func findByPidFile(pidFile string) (*os.Process, error)
+{
+    pidBuf, err := os.ReadFile(pidFile)
+    if (err != nil)
+    {
+        return nil, err
+    }
+    pid, err := strconv.ParseInt(strings.TrimSpace(string(pidBuf)), 0, 64)
+    if (err != nil)
+    {
+        return nil, err
+    }
+    proc, err := os.FindProcess(int(pid))
+    if (err != nil)
+    {
+        return nil, err
+    }
+    return proc, nil
+}
+
+func killByPidFile(pidFile string) error
+{
+    klog.Infof("killing process with PID from file %s", pidFile)
+    proc, err := findByPidFile(pidFile)
+    if (err != nil)
+    {
+        return err
+    }
+    return proc.Signal(syscall.SIGTERM)
+}
+
+func startStorageDaemon(vdpaId, volName, pidFile, configPath string, readonly bool) error
+{
+    // Start qemu-storage-daemon
+    blockSpec := map[string]interface{}{
+        "node-name": "disk1",
+        "driver": "vitastor",
+        "image": volName,
+        "cache": map[string]bool{
+            "direct": true,
+            "no-flush": false,
+        },
+        "discard": "unmap",
+    }
+    if (configPath != "")
+    {
+        blockSpec["config-path"] = configPath
+    }
+    blockSpecJson, _ := json.Marshal(blockSpec)
+    writable := "true"
+    if (readonly)
+    {
+        writable = "false"
+    }
+    _, _, err := system(
+        "/usr/bin/qemu-storage-daemon", "--daemonize", "--pidfile", pidFile, "--blockdev", string(blockSpecJson),
+        "--export", "vduse-blk,id="+vdpaId+",node-name=disk1,name="+vdpaId+",num-queues=16,queue-size=128,writable="+writable,
+    )
+    return err
+}
+
+func (ns *NodeServer) mapVduse(volName string, ctxVars map[string]string, readonly bool) (string, string, error)
+{
+    // Generate state file
+    stateFd, err := os.CreateTemp(ns.stateDir, "vitastor-vduse-*.json")
+    if (err != nil)
+    {
+        return "", "", err
+    }
+    stateFile := stateFd.Name()
+    stateFd.Close()
+    vdpaId := filepath.Base(stateFile)
+    vdpaId = vdpaId[0:len(vdpaId)-5] // remove ".json"
+    pidFile := ns.stateDir + vdpaId + ".pid"
+    // Map VDUSE device via qemu-storage-daemon
+    err = startStorageDaemon(vdpaId, volName, pidFile, ctxVars["configPath"], readonly)
+    if (err == nil)
+    {
+        // Add device to VDPA bus
+        _, _, err = system("/sbin/vdpa", "-j", "dev", "add", "name", vdpaId, "mgmtdev", "vduse")
+        if (err == nil)
+        {
+            // Find block device name
+            var matches []string
+            matches, err = filepath.Glob("/sys/bus/vdpa/devices/"+vdpaId+"/virtio*/block/*")
+            if (err == nil && len(matches) == 0)
+            {
+                err = errors.New("/sys/bus/vdpa/devices/"+vdpaId+"/virtio*/block/* is not found")
+            }
+            if (err == nil)
+            {
+                blockdev := "/dev/"+filepath.Base(matches[0])
+                _, err = os.Stat(blockdev)
+                if (err == nil)
+                {
+                    // Generate state file
+                    stateJSON, _ := json.Marshal(&DeviceState{
+                        ConfigPath: ctxVars["configPath"],
+                        VdpaId:     vdpaId,
+                        Image:      volName,
+                        Blockdev:   blockdev,
+                        Readonly:   readonly,
+                        PidFile:    pidFile,
+                    })
+                    err = os.WriteFile(stateFile, stateJSON, 0600)
+                    if (err == nil)
+                    {
+                        return blockdev, vdpaId, nil
+                    }
+                }
+            }
+        }
+        killErr := killByPidFile(pidFile)
+        if (killErr != nil)
+        {
+            klog.Errorf("Failed to kill started qemu-storage-daemon: %v", killErr)
+        }
+        os.Remove(stateFile)
+        os.Remove(pidFile)
+    }
+    return "", "", err
+}
+
+func (ns *NodeServer) unmapVduse(devicePath string)
+{
+    if (len(devicePath) < 6 || devicePath[0:6] != "/dev/v")
+    {
+        klog.Errorf("%s does not start with /dev/v", devicePath)
+        return
+    }
+    vduseDev, err := os.Readlink("/sys/block/"+devicePath[5:])
+    if (err != nil)
+    {
+        klog.Errorf("%s is not a symbolic link to VDUSE device (../devices/virtual/vduse/xxx): %v", devicePath, err)
+        return
+    }
+    vdpaId := ""
+    p := strings.Index(vduseDev, "/vduse/")
+    if (p >= 0)
+    {
+        vduseDev = vduseDev[p+7:]
+        p = strings.Index(vduseDev, "/")
+        if (p >= 0)
+        {
+            vdpaId = vduseDev[0:p]
+        }
+    }
+    if (vdpaId == "")
+    {
+        klog.Errorf("%s is not a symbolic link to VDUSE device (../devices/virtual/vduse/xxx), but is %v", devicePath, vduseDev)
+        return
+    }
+    ns.unmapVduseById(vdpaId)
+}
+
+func (ns *NodeServer) unmapVduseById(vdpaId string)
+{
+    _, err := os.Stat("/sys/bus/vdpa/devices/"+vdpaId)
+    if (err != nil)
+    {
+        klog.Errorf("failed to stat /sys/bus/vdpa/devices/"+vdpaId+": %v", err)
+    }
+    else
+    {
+        _, _, _ = system("/sbin/vdpa", "-j", "dev", "del", vdpaId)
+    }
+    stateFile := ns.stateDir + vdpaId + ".json"
+    os.Remove(stateFile)
+    pidFile := ns.stateDir + vdpaId + ".pid"
+    _, err = os.Stat(pidFile)
+    if (os.IsNotExist(err))
+    {
+        // ok, already killed
+    }
+    else if (err != nil)
+    {
+        klog.Errorf("Failed to stat %v: %v", pidFile, err)
+        return
+    }
+    else
+    {
+        err = killByPidFile(pidFile)
+        if (err != nil)
+        {
+            klog.Errorf("Failed to kill started qemu-storage-daemon: %v", err)
+        }
+        os.Remove(pidFile)
+    }
+}
+
+func (ns *NodeServer) restarter()
+{
+    // Restart dead VDUSE daemons at regular intervals
+    // Otherwise volume I/O may hang in case of a qemu-storage-daemon crash
+    // Moreover, it may lead to a kernel panic of the kernel is configured to
+    // panic on hung tasks
+    ticker := time.NewTicker(ns.restartInterval)
+    defer ticker.Stop()
+    for
+    {
+        <-ticker.C
+        ns.restoreVduseDaemons()
+    }
+}
+
+func (ns *NodeServer) restoreVduseDaemons()
+{
+    pattern := ns.stateDir+"vitastor-vduse-*.json"
+    matches, err := filepath.Glob(pattern)
+    if (err != nil)
+    {
+        klog.Errorf("failed to list %s: %v", pattern, err)
+    }
+    if (len(matches) == 0)
+    {
+        return
+    }
+    devList := make(map[string]interface{})
+    // example output: {"dev":{"test1":{"type":"block","mgmtdev":"vduse","vendor_id":0,"max_vqs":16,"max_vq_size":128}}}
+    devListJSON, _, err := system("/sbin/vdpa", "-j", "dev", "list")
+    if (err != nil)
+    {
+        return
+    }
+    err = json.Unmarshal(devListJSON, &devList)
+    devs, ok := devList["dev"].(map[string]interface{})
+    if (err != nil || !ok)
+    {
+        klog.Errorf("/sbin/vdpa -j dev list returned bad JSON (error %v): %v", err, string(devListJSON))
+        return
+    }
+    for _, stateFile := range matches
+    {
+        vdpaId := filepath.Base(stateFile)
+        vdpaId = vdpaId[0:len(vdpaId)-5]
+        // Check if VDPA device is still added to the bus
+        if (devs[vdpaId] != nil)
+        {
+            // Check if the storage daemon is still active
+            pidFile := ns.stateDir + vdpaId + ".pid"
+            exists := false
+            proc, err := findByPidFile(pidFile)
+            if (err == nil)
+            {
+                exists = proc.Signal(syscall.Signal(0)) == nil
+            }
+            if (!exists)
+            {
+                // Restart daemon
+                stateJSON, err := os.ReadFile(stateFile)
+                if (err != nil)
+                {
+                    klog.Warningf("error reading state file %v: %v", stateFile, err)
+                }
+                else
+                {
+                    var state DeviceState
+                    err := json.Unmarshal(stateJSON, &state)
+                    if (err != nil)
+                    {
+                        klog.Warningf("state file %v contains invalid JSON (error %v): %v", stateFile, err, string(stateJSON))
+                    }
+                    else
+                    {
+                        klog.Warningf("restarting storage daemon for volume %v (VDPA ID %v)", state.Image, vdpaId)
+                        _ = startStorageDaemon(vdpaId, state.Image, pidFile, state.ConfigPath, state.Readonly)
+                    }
+                }
+            }
+        }
+        else
+        {
+            // Unused, clean it up
+            ns.unmapVduseById(vdpaId)
+        }
+    }
+}
+
+// NodePublishVolume mounts the volume mounted to the staging path to the target path
+func (ns *NodeServer) NodePublishVolume(ctx context.Context, req *csi.NodePublishVolumeRequest) (*csi.NodePublishVolumeResponse, error)
+{
+    klog.Infof("received node publish volume request %+v", protosanitizer.StripSecrets(req))
+
+    targetPath := req.GetTargetPath()
+    isBlock := req.GetVolumeCapability().GetBlock() != nil
+
+    // Check that it's not already mounted
+    _, err := mount.IsNotMountPoint(ns.mounter, targetPath)
+    if (err != nil)
+    {
+        if (os.IsNotExist(err))
+        {
+            if (isBlock)
+            {
+                pathFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, 0o600)
+                if (err != nil)
+                {
+                    klog.Errorf("failed to create block device mount target %s with error: %v", targetPath, err)
+                    return nil, err
+                }
+                err = pathFile.Close()
+                if (err != nil)
+                {
+                    klog.Errorf("failed to close %s with error: %v", targetPath, err)
+                    return nil, err
+                }
+            }
+            else
+            {
+                err := os.MkdirAll(targetPath, 0777)
+                if (err != nil)
+                {
+                    klog.Errorf("failed to create fs mount target %s with error: %v", targetPath, err)
+                    return nil, err
+                }
+            }
+        }
+        else
+        {
+            return nil, err
+        }
+    }
+
+    ctxVars := make(map[string]string)
+    err = json.Unmarshal([]byte(req.VolumeId), &ctxVars)
+    if (err != nil)
+    {
+        return nil, status.Error(codes.Internal, "volume ID not in JSON format")
+    }
+    volName := ctxVars["name"]
+
+    _, err = GetConnectionParams(ctxVars)
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    var devicePath, vdpaId string
+    if (!ns.useVduse)
+    {
+        devicePath, err = ns.mapNbd(volName, ctxVars, req.GetReadonly())
+    }
+    else
+    {
+        devicePath, vdpaId, err = ns.mapVduse(volName, ctxVars, req.GetReadonly())
+    }
+    if (err != nil)
+    {
+        return nil, err
+    }
+
+    diskMounter := &mount.SafeFormatAndMount{Interface: ns.mounter, Exec: utilexec.New()}
+    if (isBlock)
+    {
+        err = diskMounter.Mount(devicePath, targetPath, "", []string{"bind"})
+    }
+    else
+    {
+        // Check existing format
+        existingFormat, err := diskMounter.GetDiskFormat(devicePath)
+        if (err != nil)
+        {
+            klog.Errorf("failed to get disk format for path %s, error: %v", err)
+            goto unmap
+        }
+
+        // Format the device (ext4 or xfs)
+        fsType := req.GetVolumeCapability().GetMount().GetFsType()
+        opt := req.GetVolumeCapability().GetMount().GetMountFlags()
+        opt = append(opt, "_netdev")
+        if ((req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY ||
+            req.VolumeCapability.AccessMode.Mode == csi.VolumeCapability_AccessMode_SINGLE_NODE_READER_ONLY) &&
+            !Contains(opt, "ro"))
+        {
+            opt = append(opt, "ro")
+        }
+        if (fsType == "xfs")
+        {
+            opt = append(opt, "nouuid")
+        }
+        readOnly := Contains(opt, "ro")
+        if (existingFormat == "" && !readOnly)
+        {
+            var cmdOut []byte
+            switch fsType
+            {
+                case "ext4":
+                    args := []string{"-m0", "-Enodiscard,lazy_itable_init=1,lazy_journal_init=1", devicePath}
+                    cmdOut, err = diskMounter.Exec.Command("mkfs.ext4", args...).CombinedOutput()
+                case "xfs":
+                    cmdOut, err = diskMounter.Exec.Command("mkfs.xfs", "-K", devicePath).CombinedOutput()
+            }
+            if (err != nil)
+            {
+                klog.Errorf("failed to run mkfs error: %v, output: %v", err, string(cmdOut))
+                goto unmap
+            }
+        }
+
+        err = diskMounter.FormatAndMount(devicePath, targetPath, fsType, opt)
+
+        // Try to run online resize on mount.
+        // FIXME: Implement online resize. It requires online resize support in vitastor-nbd.
+        if (err == nil && existingFormat != "" && !readOnly)
+        {
+            var cmdOut []byte
+            switch (fsType)
+            {
+                case "ext4":
+                    cmdOut, err = diskMounter.Exec.Command("resize2fs", devicePath).CombinedOutput()
+                case "xfs":
+                    cmdOut, err = diskMounter.Exec.Command("xfs_growfs", devicePath).CombinedOutput()
+            }
+            if (err != nil)
+            {
+                klog.Errorf("failed to run resizefs error: %v, output: %v", err, string(cmdOut))
+                goto unmap
+            }
+        }
+    }
+    if (err != nil)
+    {
+        klog.Errorf(
+            "failed to mount device path (%s) to path (%s) for volume (%s) error: %s",
+            devicePath, targetPath, volName, err,
+        )
+        goto unmap
+    }
+    return &csi.NodePublishVolumeResponse{}, nil
+
+unmap:
+    if (!ns.useVduse || len(devicePath) >= 8 && devicePath[0:8] == "/dev/nbd")
+    {
+        ns.unmapNbd(devicePath)
+    }
+    else
+    {
+        ns.unmapVduseById(vdpaId)
+    }
+    return nil, err
+}
+
+// NodeUnpublishVolume unmounts the volume from the target path
+func (ns *NodeServer) NodeUnpublishVolume(ctx context.Context, req *csi.NodeUnpublishVolumeRequest) (*csi.NodeUnpublishVolumeResponse, error)
+{
+    klog.Infof("received node unpublish volume request %+v", protosanitizer.StripSecrets(req))
+    targetPath := req.GetTargetPath()
+    devicePath, refCount, err := mount.GetDeviceNameFromMount(ns.mounter, targetPath)
+    if (err != nil)
+    {
+        if (os.IsNotExist(err))
+        {
+            return nil, status.Error(codes.NotFound, "Target path not found")
+        }
+        return nil, err
+    }
+    if (devicePath == "")
+    {
+        // volume not mounted
+        klog.Warningf("%s is not a mountpoint, deleting", targetPath)
+        os.Remove(targetPath)
+        return &csi.NodeUnpublishVolumeResponse{}, nil
+    }
+    // unmount
+    err = mount.CleanupMountPoint(targetPath, ns.mounter, false)
+    if (err != nil)
+    {
+        return nil, err
+    }
+    // unmap NBD device
+    if (refCount == 1)
+    {
+        if (!ns.useVduse)
+        {
+            ns.unmapNbd(devicePath)
+        }
+        else
+        {
+            ns.unmapVduse(devicePath)
+        }
+    }
+    return &csi.NodeUnpublishVolumeResponse{}, nil
+}
+
+// NodeGetVolumeStats returns volume capacity statistics available for the volume
+func (ns *NodeServer) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeStatsRequest) (*csi.NodeGetVolumeStatsResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// NodeExpandVolume expanding the file system on the node
+func (ns *NodeServer) NodeExpandVolume(ctx context.Context, req *csi.NodeExpandVolumeRequest) (*csi.NodeExpandVolumeResponse, error)
+{
+    return nil, status.Error(codes.Unimplemented, "")
+}
+
+// NodeGetCapabilities returns the supported capabilities of the node server
+func (ns *NodeServer) NodeGetCapabilities(ctx context.Context, req *csi.NodeGetCapabilitiesRequest) (*csi.NodeGetCapabilitiesResponse, error)
+{
+    return &csi.NodeGetCapabilitiesResponse{}, nil
+}
+
+// NodeGetInfo returns NodeGetInfoResponse for CO.
+func (ns *NodeServer) NodeGetInfo(ctx context.Context, req *csi.NodeGetInfoRequest) (*csi.NodeGetInfoResponse, error)
+{
+    klog.Infof("received node get info request %+v", protosanitizer.StripSecrets(req))
+    return &csi.NodeGetInfoResponse{
+        NodeId: ns.NodeID,
+    }, nil
+}

csi/src/server.go

@@ -0,0 +1,36 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package vitastor
+
+import (
+    "k8s.io/klog"
+)
+
+type Driver struct
+{
+    *Config
+}
+
+// NewDriver create new instance driver
+func NewDriver(config *Config) (*Driver, error)
+{
+    if (config == nil)
+    {
+        klog.Errorf("Vitastor CSI driver initialization failed")
+        return nil, nil
+    }
+    driver := &Driver{
+        Config: config,
+    }
+    klog.Infof("Vitastor CSI driver initialized")
+    return driver, nil
+}
+
+// Start server
+func (driver *Driver) Run()
+{
+    server := NewNonBlockingGRPCServer()
+    server.Start(driver.Endpoint, NewIdentityServer(driver), NewControllerServer(driver), NewNodeServer(driver))
+    server.Wait()
+}

csi/vitastor-csi.go

@@ -0,0 +1,39 @@
+// Copyright (c) Vitaliy Filippov, 2019+
+// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
+
+package main
+
+import (
+    "flag"
+    "fmt"
+    "os"
+    "k8s.io/klog"
+    "vitastor.io/csi/src"
+)
+
+func main()
+{
+    var config = vitastor.NewConfig()
+    flag.StringVar(&config.Endpoint, "endpoint", "", "CSI endpoint")
+    flag.StringVar(&config.NodeID, "node", "", "Node ID")
+    flag.Parse()
+    if (config.Endpoint == "")
+    {
+        config.Endpoint = os.Getenv("CSI_ENDPOINT")
+    }
+    if (config.NodeID == "")
+    {
+        config.NodeID = os.Getenv("NODE_ID")
+    }
+    if (config.Endpoint == "" && config.NodeID == "")
+    {
+        fmt.Fprintf(os.Stderr, "Please set -endpoint and -node / CSI_ENDPOINT & NODE_ID env vars\n")
+        os.Exit(1)
+    }
+    drv, err := vitastor.NewDriver(config)
+    if (err != nil)
+    {
+        klog.Fatalln(err)
+    }
+    drv.Run()
+}

debian/build-pve-qemu.sh

@@ -0,0 +1,58 @@
+exit
+
+git clone https://git.yourcmc.ru/vitalif/pve-qemu .
+
+# bookworm
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
+
+perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian bookworm main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget ca-certificates
+wget https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control
+
+# bullseye
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-bullseye debian:bullseye bash
+
+grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve bullseye pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian bullseye main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget
+wget https://enterprise.proxmox.com/debian/proxmox-release-bullseye.gpg -O /etc/apt/trusted.gpg.d/proxmox-release-bullseye.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control
+
+# buster
+
+docker run -it -v `pwd`/pve-qemu:/root/pve-qemu --name pve-qemu-buster debian:buster bash
+
+grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb /deb-src /' >> /etc/apt/sources.list
+echo 'deb [arch=amd64] http://download.proxmox.com/debian/pve buster pve-no-subscription' >> /etc/apt/sources.list
+echo 'deb https://vitastor.io/debian buster main' >> /etc/apt/sources.list
+echo 'deb http://deb.debian.org/debian buster-backports main' >> /etc/apt/sources.list
+echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf
+echo 'ru_RU UTF-8' >> /etc/locale.gen
+echo 'en_US UTF-8' >> /etc/locale.gen
+apt-get update
+apt-get install wget ca-certificates
+wget http://download.proxmox.com/debian/proxmox-ve-release-6.x.gpg -O /etc/apt/trusted.gpg.d/proxmox-ve-release-6.x.gpg
+wget https://vitastor.io/debian/pubkey.gpg -O /etc/apt/trusted.gpg.d/vitastor.gpg
+apt-get update
+apt-get install git devscripts equivs wget mc libjemalloc-dev vitastor-client-dev lintian locales
+mk-build-deps --install ./control

debian/build-vitastor-bookworm.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cat < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build --build-arg REL=bookworm -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-bullseye.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cat < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/build-vitastor-buster.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+cat < vitastor.Dockerfile > ../Dockerfile
+cd ..
+mkdir -p packages
+sudo podman build --build-arg REL=buster -v `pwd`/packages:/root/packages -f Dockerfile .
+rm Dockerfile

debian/changelog

@@ -0,0 +1,41 @@
+vitastor (1.4.4-1) unstable; urgency=medium
+
+  * Bugfixes
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Fri, 03 Jun 2022 02:09:44 +0300
+
+vitastor (0.7.0-1) unstable; urgency=medium
+
+  * Implement NFS proxy
+  * Add documentation
+  * Bugfixes
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sun, 29 May 2022 23:39:13 +0300
+
+vitastor (0.6.3-1) unstable; urgency=medium
+
+  * RDMA support
+  * Bugfixes
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sat, 01 May 2021 18:46:10 +0300
+
+vitastor (0.6.0-1) unstable; urgency=medium
+
+  * Snapshots and Copy-on-Write clones
+  * Image metadata in etcd (name, size)
+  * Image I/O and space statistics in etcd
+  * Write throttling for smoothing random write workloads in SSD+HDD configurations
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sun, 11 Apr 2021 00:49:18 +0300
+
+vitastor (0.5.1-1) unstable; urgency=medium
+
+  * Add jerasure support
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Sat, 05 Dec 2020 17:02:26 +0300
+
+vitastor (0.5-1) unstable; urgency=medium
+
+  * First packaging for Debian
+
+ -- Vitaliy Filippov <vitalif@yourcmc.ru>  Thu, 05 Nov 2020 02:20:59 +0300

debian/compat

@@ -0,0 +1 @@
+13

debian/control

@@ -0,0 +1,55 @@
+Source: vitastor
+Section: admin
+Priority: optional
+Maintainer: Vitaliy Filippov <vitalif@yourcmc.ru>
+Build-Depends: debhelper, liburing-dev (>= 0.6), g++ (>= 8), libstdc++6 (>= 8), linux-libc-dev, libgoogle-perftools-dev, libjerasure-dev, libgf-complete-dev, libibverbs-dev, libisal-dev, cmake, pkg-config
+Standards-Version: 4.5.0
+Homepage: https://vitastor.io/
+Rules-Requires-Root: no
+
+Package: vitastor
+Architecture: amd64
+Depends: vitastor-osd, vitastor-mon, vitastor-client, vitastor-client-dev, vitastor-fio
+Description: Vitastor, a fast software-defined clustered block storage
+ Vitastor is a small, simple and fast clustered block storage (storage for VM drives),
+ architecturally similar to Ceph which means strong consistency, primary-replication,
+ symmetric clustering and automatic data distribution over any number of drives of any
+ size with configurable redundancy (replication or erasure codes/XOR).
+
+Package: vitastor-osd
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version}), fdisk, util-linux, parted
+Description: Vitastor, a fast software-defined clustered block storage - object storage daemon
+ Vitastor object storage daemon, i.e. server program that stores data.
+
+Package: vitastor-mon
+Architecture: amd64
+Depends: ${misc:Depends}, nodejs (>= 10), node-sprintf-js, node-ws (>= 7), lp-solve
+Description: Vitastor, a fast software-defined clustered block storage - monitor
+ Vitastor monitor, i.e. server program responsible for watching cluster state and
+ scheduling cluster-level operations.
+
+Package: vitastor-client
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}
+Description: Vitastor, a fast software-defined clustered block storage - client
+ Vitastor client library and command-line interface.
+
+Package: vitastor-client-dev
+Section: devel
+Architecture: amd64
+Depends: ${misc:Depends}, vitastor-client (= ${binary:Version})
+Description: Vitastor, a fast software-defined clustered block storage - development files
+ Vitastor library headers for development.
+
+Package: vitastor-fio
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version}), fio (= ${dep:fio})
+Description: Vitastor, a fast software-defined clustered block storage - fio drivers
+ Vitastor fio drivers for benchmarking.
+
+Package: pve-storage-vitastor
+Architecture: amd64
+Depends: ${shlibs:Depends}, ${misc:Depends}, vitastor-client (= ${binary:Version})
+Description: Vitastor Proxmox Virtual Environment storage plugin
+ Vitastor storage plugin for Proxmox Virtual Environment.

debian/copyright

@@ -0,0 +1,21 @@
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: vitastor
+Upstream-Contact: Vitaliy Filippov <vitalif@yourcmc.ru>
+Source: https://vitastor.io
+
+Files: *
+Copyright: 2019+ Vitaliy Filippov <vitalif@yourcmc.ru>
+License: Multiple licenses VNPL-1.1 and/or GPL-2.0+
+ All server-side code (OSD, Monitor and so on) is licensed under the terms of
+ Vitastor Network Public License 1.1 (VNPL 1.1), a copyleft license based on
+ GNU GPLv3.0 with the additional "Network Interaction" clause which requires
+ opensourcing all programs directly or indirectly interacting with Vitastor
+ through a computer network and expressly designed to be used in conjunction
+ with it ("Proxy Programs"). Proxy Programs may be made public not only under
+ the terms of the same license, but also under the terms of any GPL-Compatible
+ Free Software License, as listed by the Free Software Foundation.
+ This is a stricter copyleft license than the Affero GPL.
+ .
+ Client libraries (cluster_client and so on) are dual-licensed under the same
+ VNPL 1.1 and also GNU GPL 2.0 or later to allow for compatibility with GPLed
+ software like QEMU and fio.

debian/fio_version

@@ -0,0 +1 @@
+dep:fio=3.16-1

debian/install

@@ -0,0 +1,4 @@
+VNPL-1.1.txt usr/share/doc/vitastor
+GPL-2.0.txt usr/share/doc/vitastor
+README.md usr/share/doc/vitastor
+README-ru.md usr/share/doc/vitastor

debian/libisal.pc

@@ -0,0 +1,11 @@
+prefix=/usr
+exec_prefix=${prefix}
+libdir=${prefix}/lib/x86_64-linux-gnu
+includedir=${prefix}/include
+
+Name: libisal
+Description: Library for storage systems
+Version: 2.30.0
+Libs: -L${libdir} -lisal
+Libs.private:
+Cflags: -I${includedir}

debian/libvirt.Dockerfile

@@ -0,0 +1,40 @@
+# Build patched libvirt for Debian Buster or Bullseye/Sid inside a container
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/libvirt.Dockerfile .
+
+ARG REL=
+FROM debian:$REL
+ARG REL=
+
+WORKDIR /root
+
+RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
+        echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        echo >> /etc/apt/preferences; \
+        echo 'Package: *' >> /etc/apt/preferences; \
+        echo "Pin: release a=$REL-backports" >> /etc/apt/preferences; \
+        echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update; apt-get -y install devscripts
+RUN apt-get -y build-dep libvirt0
+RUN apt-get -y install libglusterfs-dev
+RUN apt-get --download-only source libvirt
+
+ADD patches/libvirt-5.0-vitastor.diff patches/libvirt-7.0-vitastor.diff patches/libvirt-7.5-vitastor.diff patches/libvirt-7.6-vitastor.diff /root
+RUN set -e; \
+    mkdir -p /root/packages/libvirt-$REL; \
+    rm -rf /root/packages/libvirt-$REL/*; \
+    cd /root/packages/libvirt-$REL; \
+    dpkg-source -x /root/libvirt*.dsc; \
+    D=$(ls -d libvirt-*/); \
+    V=$(ls -d libvirt-*/ | perl -pe 's/libvirt-(\d+\.\d+).*/$1/'); \
+    cp /root/libvirt-$V-vitastor.diff $D/debian/patches; \
+    echo libvirt-$V-vitastor.diff >> $D/debian/patches/series; \
+    cd $D; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?(\+deb[u\d]+)?\).*$/$1/')+vitastor2; \
+    DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Add Vitastor support'; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/libvirt-$REL/$D

debian/patched-qemu.Dockerfile

@@ -0,0 +1,61 @@
+# Build patched QEMU for Debian inside a container
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/patched-qemu.Dockerfile .
+
+ARG REL=
+FROM debian:$REL
+ARG REL=
+
+WORKDIR /root
+
+RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" -o "$REL" = "bookworm" ]; then \
+        echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        echo >> /etc/apt/preferences; \
+        echo 'Package: *' >> /etc/apt/preferences; \
+        echo "Pin: release a=$REL-backports" >> /etc/apt/preferences; \
+        echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update
+RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y build-dep qemu
+# To build a custom version
+#RUN cp /root/packages/qemu-orig/* /root
+RUN apt-get --download-only source qemu
+
+ADD patches /root/vitastor/patches
+ADD src/qemu_driver.c /root/vitastor/src/qemu_driver.c
+
+#RUN set -e; \
+#    apt-get install -y wget; \
+#    wget -q -O /etc/apt/trusted.gpg.d/vitastor.gpg https://vitastor.io/debian/pubkey.gpg; \
+#    (echo deb http://vitastor.io/debian $REL main > /etc/apt/sources.list.d/vitastor.list); \
+#    (echo "APT::Install-Recommends false;" > /etc/apt/apt.conf) && \
+#    apt-get update; \
+#    apt-get install -y vitastor-client vitastor-client-dev quilt
+
+RUN set -e; \
+    dpkg -i /root/packages/vitastor-$REL/vitastor-client_*.deb /root/packages/vitastor-$REL/vitastor-client-dev_*.deb; \
+    apt-get update; \
+    apt-get install -y quilt; \
+    mkdir -p /root/packages/qemu-$REL; \
+    rm -rf /root/packages/qemu-$REL/*; \
+    cd /root/packages/qemu-$REL; \
+    dpkg-source -x /root/qemu*.dsc; \
+    QEMU_VER=$(ls -d qemu*/ | perl -pe 's!^.*?(\d+\.\d+).*!$1!'); \
+    D=$(ls -d qemu*/); \
+    cp /root/vitastor/patches/qemu-$QEMU_VER-vitastor.patch ./qemu-*/debian/patches; \
+    echo qemu-$QEMU_VER-vitastor.patch >> $D/debian/patches/series; \
+    cd /root/packages/qemu-$REL/qemu-*/; \
+    quilt push -a; \
+    quilt add block/vitastor.c; \
+    cp /root/vitastor/src/qemu_driver.c block/vitastor.c; \
+    quilt refresh; \
+    V=$(head -n1 debian/changelog | perl -pe 's/5\.2\+dfsg-9/5.2+dfsg-11/; s/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor4; \
+    if [ "$REL" = bullseye ]; then V=${V}bullseye; fi; \
+    DEBEMAIL="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v $V 'Plug Vitastor block driver'; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/qemu-$REL/qemu-*/

debian/pve-storage-vitastor.install

@@ -0,0 +1 @@
+patches/VitastorPlugin.pm usr/share/perl5/PVE/Storage/Custom/

debian/raw.h

@@ -0,0 +1,19 @@
+/* Removed in Linux 5.14 */
+
+/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */
+#ifndef __LINUX_RAW_H
+#define __LINUX_RAW_H
+
+#include <linux/types.h>
+
+#define RAW_SETBIND	_IO( 0xac, 0 )
+#define RAW_GETBIND	_IO( 0xac, 1 )
+
+struct raw_config_request 
+{
+	int	raw_minor;
+	__u64	block_major;
+	__u64	block_minor;
+};
+
+#endif /* __LINUX_RAW_H */

debian/rules

@@ -0,0 +1,10 @@
+#!/usr/bin/make -f
+export DH_VERBOSE = 1
+
+%:
+	dh $@
+
+override_dh_installdeb:
+	cat debian/fio_version >> debian/vitastor-fio.substvars
+	[ -f debian/qemu_version ] && (cat debian/qemu_version >> debian/vitastor-qemu.substvars) || true
+	dh_installdeb

debian/source/format

@@ -0,0 +1 @@
+3.0 (quilt)

debian/vitastor-client-dev.install

@@ -0,0 +1,2 @@
+usr/include
+usr/lib/*/pkgconfig

debian/vitastor-client.install

@@ -0,0 +1,6 @@
+usr/bin/vita
+usr/bin/vitastor-cli
+usr/bin/vitastor-rm
+usr/bin/vitastor-nbd
+usr/bin/vitastor-nfs
+usr/lib/*/libvitastor*.so*

debian/vitastor-fio.install

@@ -0,0 +1 @@
+usr/lib/*/libfio*.so*

debian/vitastor-mon.install

@@ -0,0 +1,2 @@
+mon usr/lib/vitastor
+mon/vitastor-mon.service /lib/systemd/system

debian/vitastor-mon.postinst

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" = "configure" ]; then
+	addgroup --system --quiet vitastor
+	adduser --system --quiet --ingroup vitastor --no-create-home --home /nonexistent vitastor
+	mkdir -p /etc/vitastor
+fi

debian/vitastor-osd.install

@@ -0,0 +1,6 @@
+usr/bin/vitastor-osd
+usr/bin/vitastor-disk
+usr/bin/vitastor-dump-journal
+mon/vitastor-osd@.service /lib/systemd/system
+mon/vitastor.target /lib/systemd/system
+mon/90-vitastor.rules /lib/udev/rules.d

debian/vitastor-osd.postinst

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+if [ "$1" = "configure" ]; then
+	addgroup --system --quiet vitastor
+	adduser --system --quiet --ingroup vitastor --no-create-home --home /nonexistent vitastor
+	install -o vitastor -g vitastor -d /var/log/vitastor
+	mkdir -p /etc/vitastor
+fi

debian/vitastor.Dockerfile

@@ -0,0 +1,57 @@
+# Build Vitastor packages for Debian inside a container
+# cd ..; podman build --build-arg REL=bullseye -v `pwd`/packages:/root/packages -f debian/vitastor.Dockerfile .
+
+ARG REL=
+FROM debian:$REL
+ARG REL=
+
+WORKDIR /root
+
+RUN if [ "$REL" = "buster" -o "$REL" = "bullseye" ]; then \
+        echo "deb http://deb.debian.org/debian $REL-backports main" >> /etc/apt/sources.list; \
+        echo >> /etc/apt/preferences; \
+        echo 'Package: *' >> /etc/apt/preferences; \
+        echo "Pin: release a=$REL-backports" >> /etc/apt/preferences; \
+        echo 'Pin-Priority: 500' >> /etc/apt/preferences; \
+    fi; \
+    grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \
+    perl -i -pe 's/Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/debian.sources || true; \
+    echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \
+    echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf
+
+RUN apt-get update
+RUN apt-get -y install fio liburing-dev libgoogle-perftools-dev devscripts
+RUN apt-get -y build-dep fio
+RUN apt-get --download-only source fio
+RUN apt-get update && apt-get -y install libjerasure-dev cmake libibverbs-dev libisal-dev
+
+ADD . /root/vitastor
+RUN set -e -x; \
+    [ -e /usr/lib/x86_64-linux-gnu/pkgconfig/libisal.pc ] || cp /root/vitastor/debian/libisal.pc /usr/lib/x86_64-linux-gnu/pkgconfig; \
+    mkdir -p /root/fio-build/; \
+    cd /root/fio-build/; \
+    rm -rf /root/fio-build/*; \
+    dpkg-source -x /root/fio*.dsc; \
+    mkdir -p /root/packages/vitastor-$REL; \
+    rm -rf /root/packages/vitastor-$REL/*; \
+    cd /root/packages/vitastor-$REL; \
+    cp -r /root/vitastor vitastor-1.4.4; \
+    cd vitastor-1.4.4; \
+    ln -s /root/fio-build/fio-*/ ./fio; \
+    FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \
+    sh copy-fio-includes.sh; \
+    rm fio; \
+    mkdir -p a b debian/patches; \
+    mv fio-copy b/fio; \
+    diff -NaurpbB a b > debian/patches/fio-headers.patch || true; \
+    echo fio-headers.patch >> debian/patches/series; \
+    rm -rf a b; \
+    echo "dep:fio=$FIO" > debian/fio_version; \
+    cd /root/packages/vitastor-$REL; \
+    tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_1.4.4.orig.tar.xz vitastor-1.4.4; \
+    cd vitastor-1.4.4; \
+    V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \
+    DEBFULLNAME="Vitaliy Filippov <vitalif@yourcmc.ru>" dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \
+    DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \
+    rm -rf /root/packages/vitastor-$REL/vitastor-*/

docker/Dockerfile

@@ -0,0 +1,9 @@
+# Build Docker image with Vitastor packages
+
+FROM debian:bullseye
+
+ADD vitastor.list /etc/apt/sources.list.d
+ADD vitastor.gpg /etc/apt/trusted.gpg.d
+ADD vitastor.pref /etc/apt/preferences.d
+ADD apt.conf /etc/apt/
+RUN apt-get update && apt-get -y install vitastor qemu-system-x86 qemu-system-common && apt-get clean

docker/apt.conf

@@ -0,0 +1 @@
+APT::Install-Recommends false;

docker/vitastor.list

@@ -0,0 +1 @@
+deb http://vitastor.io/debian bullseye main

docker/vitastor.pref

@@ -0,0 +1,3 @@
+Package: *
+Pin: origin "vitastor.io"
+Pin-Priority: 1000

docs/config.en.md

@@ -0,0 +1,40 @@
+[Documentation](../README.md#documentation) → Configuration Reference
+
+-----
+
+[Читать на русском](config.ru.md)
+
+# Configuration Reference
+
+Vitastor configuration consists of:
+- [Configuration parameters (key-value)](#parameter-reference)
+- [Pool configuration](config/pool.en.md)
+- [OSD placement tree configuration](config/pool.en.md#placement-tree)
+- [Separate OSD settings](config/pool.en.md#osd-settings)
+- [Inode configuration](config/inode.en.md) i.e. image metadata like name, size and parent reference
+
+Configuration parameters can be set in 3 places:
+- Configuration file (`/etc/vitastor/vitastor.conf` or other path)
+- etcd key `/vitastor/config/global`. Most variables can be set there, but etcd
+  connection parameters should obviously be set in the configuration file.
+- Command line of Vitastor components: OSD (when you run it without vitastor-disk),
+  mon, fio and QEMU options, OpenStack/Proxmox/etc configuration. The latter
+  doesn't allow to set all variables directly, but it allows to override the
+  configuration file and set everything you need inside it.
+- OSD superblocks created by [vitastor-disk](usage/disk.en.md) contain
+  primarily disk layout parameters of specific OSDs. In fact, these parameters
+  are automatically passed into the command line of vitastor-osd process, so
+  they have the same "status" as command-line parameters.
+
+In the future, additional configuration methods may be added:
+- OSD-specific keys in etcd like `/vitastor/config/osd/<number>`.
+
+## Parameter Reference
+
+- [Common](config/common.en.md)
+- [Network](config/network.en.md)
+- [Client](config/client.en.md)
+- [Global Disk Layout](config/layout-cluster.en.md)
+- [OSD Disk Layout](config/layout-osd.en.md)
+- [OSD Runtime Parameters](config/osd.en.md)
+- [Monitor](config/monitor.en.md)

docs/config.ru.md

@@ -0,0 +1,43 @@
+[Документация](../README-ru.md#документация) → Конфигурация Vitastor
+
+-----
+
+[Read in English](config.en.md)
+
+# Конфигурация Vitastor
+
+Конфигурация Vitastor состоит из:
+- [Параметров (ключ-значение)](#список-параметров)
+- [Настроек пулов](config/pool.ru.md)
+- [Настроек дерева OSD](config/pool.ru.md#дерево-размещения)
+- [Настроек отдельных OSD](config/pool.ru.md#настройки-osd)
+- [Настроек инодов](config/inode.ru.md), т.е. метаданных образов, таких, как имя, размер и ссылки на
+  родительский образ
+
+Параметры конфигурации могут задаваться в 3 местах:
+- Файле конфигурации (`/etc/vitastor/vitastor.conf` или по другому пути)
+- Ключе в etcd `/vitastor/config/global`. Большая часть параметров может
+  задаваться там, кроме, естественно, самих параметров соединения с etcd,
+  которые должны задаваться в файле конфигурации
+- В командной строке компонентов Vitastor: OSD (при ручном запуске без vitastor-disk),
+  монитора, опциях fio и QEMU, настроек OpenStack, Proxmox и т.п. Последние,
+  как правило, не включают полный набор параметров напрямую, но позволяют
+  определить путь к файлу конфигурации и задать любые параметры в нём.
+- В суперблоке OSD, записываемом [vitastor-disk](usage/disk.ru.md) - параметры,
+  связанные с дисковым форматом и с этим конкретным OSD. На самом деле,
+  при запуске OSD эти параметры автоматически передаются в командную строку
+  процесса vitastor-osd, то есть по "статусу" они эквивалентны параметрам
+  командной строки OSD.
+
+В будущем также могут быть добавлены другие способы конфигурации:
+- OSD-специфичные ключи в etcd типа `/vitastor/config/osd/<номер>`.
+
+## Список параметров
+
+- [Общие](config/common.ru.md)
+- [Сеть](config/network.ru.md)
+- [Клиентский код](config/client.ru.md)
+- [Глобальные дисковые параметры](config/layout-cluster.ru.md)
+- [Дисковые параметры OSD](config/layout-osd.ru.md)
+- [Прочие параметры OSD](config/osd.ru.md)
+- [Параметры мониторов](config/monitor.ru.md)

docs/config/client.en.md

@@ -0,0 +1,137 @@
+[Documentation](../../README.md#documentation) → [Configuration](../config.en.md) → Client Parameters
+
+-----
+
+[Читать на русском](client.ru.md)
+
+# Client Parameters
+
+These parameters apply only to Vitastor clients (QEMU, fio, NBD and so on) and
+affect their interaction with the cluster.
+
+- [client_max_dirty_bytes](#client_max_dirty_bytes)
+- [client_max_dirty_ops](#client_max_dirty_ops)
+- [client_enable_writeback](#client_enable_writeback)
+- [client_max_buffered_bytes](#client_max_buffered_bytes)
+- [client_max_buffered_ops](#client_max_buffered_ops)
+- [client_max_writeback_iodepth](#client_max_writeback_iodepth)
+- [nbd_timeout](#nbd_timeout)
+- [nbd_max_devices](#nbd_max_devices)
+- [nbd_max_part](#nbd_max_part)
+
+## client_max_dirty_bytes
+
+- Type: integer
+- Default: 33554432
+- Can be changed online: yes
+
+Without [immediate_commit](layout-cluster.en.md#immediate_commit)=all this parameter sets the limit of "dirty"
+(not committed by fsync) data allowed by the client before forcing an
+additional fsync and committing the data. Also note that the client always
+holds a copy of uncommitted data in memory so this setting also affects
+RAM usage of clients.
+
+## client_max_dirty_ops
+
+- Type: integer
+- Default: 1024
+- Can be changed online: yes
+
+Same as client_max_dirty_bytes, but instead of total size, limits the number
+of uncommitted write operations.
+
+## client_enable_writeback
+
+- Type: boolean
+- Default: false
+- Can be changed online: yes
+
+This parameter enables client-side write buffering. This means that write
+requests are accumulated in memory for a short time before being sent to
+a Vitastor cluster which allows to send them in parallel and increase
+performance of some applications. Writes are buffered until client forces
+a flush with fsync() or until the amount of buffered writes exceeds the
+limit.
+
+Write buffering significantly increases performance of some applications,
+for example, CrystalDiskMark under Windows (LOL :-D), but also any other
+applications if they do writes in one of two non-optimal ways: either if
+they do a lot of small (4 kb or so) sequential writes, or if they do a lot
+of small random writes, but without any parallelism or asynchrony, and also
+without calling fsync().
+
+With write buffering enabled, you can expect around 22000 T1Q1 random write
+iops in QEMU more or less regardless of the quality of your SSDs, and this
+number is in fact bound by QEMU itself rather than Vitastor (check it
+yourself by adding a "driver=null-co" disk in QEMU). Without write
+buffering, the current record is 9900 iops, but the number is usually
+even lower with non-ideal hardware, for example, it may be 5000 iops.
+
+Even when this parameter is enabled, write buffering isn't enabled until
+the client explicitly allows it, because enabling it without the client
+being aware of the fact that his writes may be buffered may lead to data
+loss. Because of this, older versions of clients don't support write
+buffering at all, newer versions of the QEMU driver allow write buffering
+only if it's enabled in disk settings with `-blockdev cache.direct=false`,
+and newer versions of FIO only allow write buffering if you don't specify
+`-direct=1`. NBD and NFS drivers allow write buffering by default.
+
+You can overcome this restriction too with the `client_writeback_allowed`
+parameter, but you shouldn't do that unless you **really** know what you
+are doing.
+
+## client_max_buffered_bytes
+
+- Type: integer
+- Default: 33554432
+- Can be changed online: yes
+
+Maximum total size of buffered writes which triggers write-back when reached.
+
+## client_max_buffered_ops
+
+- Type: integer
+- Default: 1024
+- Can be changed online: yes
+
+Maximum number of buffered writes which triggers write-back when reached.
+Multiple consecutive modified data regions are counted as 1 write here.
+
+## client_max_writeback_iodepth
+
+- Type: integer
+- Default: 256
+- Can be changed online: yes
+
+Maximum number of parallel writes when flushing buffered data to the server.
+
+## nbd_timeout
+
+- Type: seconds
+- Default: 300
+
+Timeout for I/O operations for [NBD](../usage/nbd.en.md). If an operation
+executes for longer than this timeout, including when your cluster is just
+temporarily down for more than timeout, the NBD device will detach by itself
+(and possibly break the mounted file system).
+
+You can set timeout to 0 to never detach, but in that case you won't be
+able to remove the kernel device at all if the NBD process dies - you'll have
+to reboot the host.
+
+## nbd_max_devices
+
+- Type: integer
+- Default: 64
+
+Maximum number of NBD devices in the system. This value is passed as
+`nbds_max` parameter for the nbd kernel module when vitastor-nbd autoloads it.
+
+## nbd_max_part
+
+- Type: integer
+- Default: 3
+
+Maximum number of partitions per NBD device. This value is passed as
+`max_part` parameter for the nbd kernel module when vitastor-nbd autoloads it.
+Note that (nbds_max)*(1+max_part) usually can't exceed 256.

docs/config/client.ru.md

@@ -0,0 +1,137 @@
+[Документация](../../README-ru.md#документация) → [Конфигурация](../config.ru.md) → Параметры клиентского кода
+
+-----
+
+[Read in English](client.en.md)
+
+# Параметры клиентского кода
+
+Данные параметры применяются только к клиентам Vitastor (QEMU, fio, NBD и т.п.) и
+затрагивают логику их работы с кластером.
+
+- [client_max_dirty_bytes](#client_max_dirty_bytes)
+- [client_max_dirty_ops](#client_max_dirty_ops)
+- [client_enable_writeback](#client_enable_writeback)
+- [client_max_buffered_bytes](#client_max_buffered_bytes)
+- [client_max_buffered_ops](#client_max_buffered_ops)
+- [client_max_writeback_iodepth](#client_max_writeback_iodepth)
+- [nbd_timeout](#nbd_timeout)
+- [nbd_max_devices](#nbd_max_devices)
+- [nbd_max_part](#nbd_max_part)
+
+## client_max_dirty_bytes
+
+- Тип: целое число
+- Значение по умолчанию: 33554432
+- Можно менять на лету: да
+
+При работе без [immediate_commit](layout-cluster.ru.md#immediate_commit)=all - это лимит объёма "грязных" (не
+зафиксированных fsync-ом) данных, при достижении которого клиент будет
+принудительно вызывать fsync и фиксировать данные. Также стоит иметь в виду,
+что в этом случае до момента fsync клиент хранит копию незафиксированных
+данных в памяти, то есть, настройка влияет на потребление памяти клиентами.
+
+## client_max_dirty_ops
+
+- Тип: целое число
+- Значение по умолчанию: 1024
+- Можно менять на лету: да
+
+Аналогично client_max_dirty_bytes, но ограничивает количество
+незафиксированных операций записи вместо их общего объёма.
+
+## client_enable_writeback
+
+- Тип: булево (да/нет)
+- Значение по умолчанию: false
+- Можно менять на лету: да
+
+Данный параметр разрешает включать буферизацию записи в памяти. Буферизация
+означает, что операции записи отправляются на кластер Vitastor не сразу, а
+могут небольшое время накапливаться в памяти и сбрасываться сразу пакетами,
+до тех пор, пока либо не будет превышен лимит неотправленных записей, либо
+пока клиент не вызовет fsync.
+
+Буферизация значительно повышает производительность некоторых приложений,
+например, CrystalDiskMark в Windows (ха-ха :-D), но также и любых других,
+которые пишут на диск неоптимально: либо последовательно, но мелкими блоками
+(например, по 4 кб), либо случайно, но без параллелизма и без fsync - то
+есть, например, отправляя 128 операций записи в разные места диска, но не
+все сразу с помощью асинхронного I/O, а по одной.
+
+В QEMU с буферизацией записи можно ожидать показателя примерно 22000
+операций случайной записи в секунду в 1 поток и с глубиной очереди 1 (T1Q1)
+без fsync, почти вне зависимости от того, насколько хороши ваши диски - эта
+цифра упирается в сам QEMU. Без буферизации рекорд пока что - 9900 операций
+в секунду, но на железе похуже может быть и поменьше, например, 5000 операций
+в секунду.
+
+При этом, даже если данный параметр включён, буферизация не включается, если
+явно не разрешена клиентом, т.к. если клиент не знает, что запросы записи
+буферизуются, это может приводить к потере данных. Поэтому в старых версиях
+клиентских драйверов буферизация записи не включается вообще, в новых
+версиях QEMU-драйвера включается, только если разрешена опцией диска
+`-blockdev cache.direct=false`, а в fio - только если нет опции `-direct=1`.
+В NBD и NFS драйверах буферизация записи разрешена по умолчанию.
+
+Можно обойти и это ограничение с помощью параметра `client_writeback_allowed`,
+но делать так не надо, если только вы не уверены в том, что делаете, на все
+100%. :-)
+
+## client_max_buffered_bytes
+
+- Тип: целое число
+- Значение по умолчанию: 33554432
+- Можно менять на лету: да
+
+Максимальный общий размер буферизованных записей, при достижении которого
+начинается процесс сброса данных на сервер.
+
+## client_max_buffered_ops
+
+- Тип: целое число
+- Значение по умолчанию: 1024
+- Можно менять на лету: да
+
+Максимальное количество буферизованных записей, при достижении которого
+начинается процесс сброса данных на сервер. При этом несколько
+последовательных изменённых областей здесь считаются 1 записью.
+
+## client_max_writeback_iodepth
+
+- Тип: целое число
+- Значение по умолчанию: 256
+- Можно менять на лету: да
+
+Максимальное число параллельных операций записи при сбросе буферов на сервер.
+
+## nbd_timeout
+
+- Тип: секунды
+- Значение по умолчанию: 300
+
+Таймаут для операций чтения/записи через [NBD](../usage/nbd.ru.md). Если
+операция выполняется дольше таймаута, включая временную недоступность
+кластера на время, большее таймаута, NBD-устройство отключится само собой
+(и, возможно, сломает примонтированную ФС).
+
+Вы можете установить таймаут в 0, чтобы никогда не отключать устройство по
+таймауту, но в этом случае вы вообще не сможете удалить устройство, если
+процесс NBD умрёт - вам придётся перезагружать сервер.
+
+## nbd_max_devices
+
+- Тип: целое число
+- Значение по умолчанию: 64
+
+Максимальное число NBD-устройств в системе. Данное значение передаётся
+модулю ядра nbd как параметр `nbds_max`, когда его загружает vitastor-nbd.
+
+## nbd_max_part
+
+- Тип: целое число
+- Значение по умолчанию: 3
+
+Максимальное число разделов на одном NBD-устройстве. Данное значение передаётся
+модулю ядра nbd как параметр `max_part`, когда его загружает vitastor-nbd.
+Имейте в виду, что (nbds_max)*(1+max_part) обычно не может превышать 256.

docs/config/common.en.md

@@ -0,0 +1,52 @@
+[Documentation](../../README.md#documentation) → [Configuration](../config.en.md) → Common Parameters
+
+-----
+
+[Читать на русском](common.ru.md)
+
+# Common Parameters
+
+These are the most common parameters which apply to all components of Vitastor.
+
+- [config_path](#config_path)
+- [etcd_address](#etcd_address)
+- [etcd_prefix](#etcd_prefix)
+- [log_level](#log_level)
+
+## config_path
+
+- Type: string
+- Default: /etc/vitastor/vitastor.conf
+
+Path to the JSON configuration file. Configuration file is optional,
+a non-existing configuration file does not prevent Vitastor from
+running if required parameters are specified.
+
+## etcd_address
+
+- Type: string or array of strings
+- Can be changed online: yes
+
+etcd connection endpoint(s). Multiple endpoints may be delimited by "," or
+specified in a JSON array `["10.0.115.10:2379/v3","10.0.115.11:2379/v3"]`.
+Note that https is not supported for etcd connections yet.
+
+etcd connection endpoints can be changed online by updating global
+configuration in etcd itself - this allows to switch the cluster to new
+etcd addresses without downtime.
+
+## etcd_prefix
+
+- Type: string
+- Default: /vitastor
+
+Prefix for all keys in etcd used by Vitastor. You can change prefix and, for
+example, use a single etcd cluster for multiple Vitastor clusters.
+
+## log_level
+
+- Type: integer
+- Default: 0
+- Can be changed online: yes
+
+Log level. Raise if you want more verbose output.

docs/config/common.ru.md

@@ -0,0 +1,50 @@
+[Документация](../../README-ru.md#документация) → [Конфигурация](../config.ru.md) → Общие параметры
+
+-----
+
+[Read in English](common.en.md)
+
+# Общие параметры
+
+Это наиболее общие параметры, используемые всеми компонентами Vitastor.
+
+- [config_path](#config_path)
+- [etcd_address](#etcd_address)
+- [etcd_prefix](#etcd_prefix)
+- [log_level](#log_level)
+
+## config_path
+
+- Тип: строка
+- Значение по умолчанию: /etc/vitastor/vitastor.conf
+
+Путь к файлу конфигурации в формате JSON. Файл конфигурации необязателен,
+без него Vitastor тоже будет работать, если переданы необходимые параметры.
+
+## etcd_address
+
+- Тип: строка или массив строк
+- Можно менять на лету: да
+
+Адрес(а) подключения к etcd. Несколько адресов могут разделяться запятой
+или указываться в виде JSON-массива `["10.0.115.10:2379/v3","10.0.115.11:2379/v3"]`.
+
+Адреса подключения к etcd можно поменять на лету, обновив конфигурацию в
+самом etcd - это позволяет переключить кластер на новые etcd без остановки.
+
+## etcd_prefix
+
+- Тип: строка
+- Значение по умолчанию: /vitastor
+
+Префикс для ключей etcd, которые использует Vitastor. Вы можете задать другой
+префикс, например, чтобы запустить несколько кластеров Vitastor с одним
+кластером etcd.
+
+## log_level
+
+- Тип: целое число
+- Значение по умолчанию: 0
+- Можно менять на лету: да
+
+Уровень логгирования. Повысьте, если хотите более подробный вывод.