diff --git a/.gitea/workflows/buildenv.Dockerfile b/.gitea/workflows/buildenv.Dockerfile new file mode 100644 index 00000000..4e4ae6d1 --- /dev/null +++ b/.gitea/workflows/buildenv.Dockerfile @@ -0,0 +1,36 @@ +FROM node:16-bullseye + +WORKDIR /root + +ADD ./docker/vitastor.gpg /etc/apt/trusted.gpg.d + +RUN echo 'deb http://deb.debian.org/debian bullseye-backports main' >> /etc/apt/sources.list; \ + echo 'deb http://vitastor.io/debian bullseye main' >> /etc/apt/sources.list; \ + echo >> /etc/apt/preferences; \ + echo 'Package: *' >> /etc/apt/preferences; \ + echo 'Pin: release a=bullseye-backports' >> /etc/apt/preferences; \ + echo 'Pin-Priority: 500' >> /etc/apt/preferences; \ + echo >> /etc/apt/preferences; \ + echo 'Package: *' >> /etc/apt/preferences; \ + echo 'Pin: origin "vitastor.io"' >> /etc/apt/preferences; \ + echo 'Pin-Priority: 1000' >> /etc/apt/preferences; \ + grep '^deb ' /etc/apt/sources.list | perl -pe 's/^deb/deb-src/' >> /etc/apt/sources.list; \ + echo 'APT::Install-Recommends false;' >> /etc/apt/apt.conf; \ + echo 'APT::Install-Suggests false;' >> /etc/apt/apt.conf + +RUN apt-get update +RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \ + liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev +RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'` +RUN apt-get -y install jq lp-solve sudo +RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'` + +RUN set -ex; \ + mkdir qemu-build; \ + cd qemu-build; \ + dpkg-source -x /root/qemu*.dsc; \ + cd qemu*/; \ + debian/rules configure-qemu || debian/rules b/configure-stamp; \ + cd b/qemu; \ + make -j8 config-poison.h || true; \ + make -j8 qapi/qapi-builtin-types.h diff --git a/.gitea/workflows/test.Dockerfile b/.gitea/workflows/test.Dockerfile new file mode 100644 index 00000000..736daafe --- /dev/null +++ b/.gitea/workflows/test.Dockerfile @@ -0,0 +1,16 @@ +FROM git.yourcmc.ru/vitalif/vitastor/buildenv + +ADD . /root/vitastor + +RUN set -e -x; \ + mkdir -p /root/fio-build/; \ + cd /root/fio-build/; \ + dpkg-source -x /root/fio*.dsc; \ + cd /root/vitastor; \ + ln -s /root/fio-build/fio-*/ ./fio; \ + ln -s /root/qemu-build/qemu-*/ ./qemu; \ + ls /usr/include/linux/raw.h || cp ./debian/raw.h /usr/include/linux/raw.h; \ + mkdir build; \ + cd build; \ + cmake .. -DWITH_ASAN=yes -DWITH_QEMU=yes; \ + make -j16 diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml new file mode 100644 index 00000000..6c8c01df --- /dev/null +++ b/.gitea/workflows/test.yml @@ -0,0 +1,552 @@ +name: Test + +on: + push: + branches: + - '*' + paths: + - '.gitea/**' + - 'src/**' + - 'mon/**' + - 'json11' + - 'cpp-btree' + - 'tests/**' + +env: + BUILDENV_IMAGE: git.yourcmc.ru/vitalif/vitastor/buildenv + TEST_IMAGE: git.yourcmc.ru/vitalif/vitastor/test + OSD_ARGS: '--etcd_quick_timeout 2000' + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + + buildenv: + runs-on: ubuntu-latest + container: git.yourcmc.ru/vitalif/gitea-ci-dind + steps: + - uses: actions/checkout@v3 + + - name: Build and push + run: | + set -ex + if ! docker manifest inspect $BUILDENV_IMAGE >/dev/null; then + docker build -t $BUILDENV_IMAGE -f .gitea/workflows/buildenv.Dockerfile . + docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}" + docker push $BUILDENV_IMAGE + fi + + build: + runs-on: ubuntu-latest + needs: buildenv + container: git.yourcmc.ru/vitalif/gitea-ci-dind + steps: + - uses: actions/checkout@v3 + with: + submodules: true + + - name: Build and push + run: | + set -ex + if ! docker manifest inspect $TEST_IMAGE:$GITHUB_SHA >/dev/null; then + docker build -t $TEST_IMAGE:$GITHUB_SHA -f .gitea/workflows/test.Dockerfile . + docker login git.yourcmc.ru -u vitalif -p "${{secrets.TOKEN}}" + docker push $TEST_IMAGE:$GITHUB_SHA + fi + + make_test: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + # leak sanitizer sometimes crashes + - run: cd /root/vitastor/build && ASAN_OPTIONS=detect_leaks=0 make -j16 test + + test_add_osd: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_add_osd.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_cas: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_cas.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_change_pg_count: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_change_pg_count.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_change_pg_count_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: SCHEME=ec /root/vitastor/tests/test_change_pg_count.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_change_pg_size: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_change_pg_size.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_create_nomaxid: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_create_nomaxid.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_etcd_fail: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: /root/vitastor/tests/test_etcd_fail.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_failure_domain: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_failure_domain.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_interrupted_rebalance: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: /root/vitastor/tests/test_interrupted_rebalance.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_interrupted_rebalance_imm: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_interrupted_rebalance_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: SCHEME=ec /root/vitastor/tests/test_interrupted_rebalance.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_interrupted_rebalance_ec_imm: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_interrupted_rebalance.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_minsize_1: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_minsize_1.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_move_reappear: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_move_reappear.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_rebalance_verify: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_rebalance_verify.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_rebalance_verify_imm: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_rebalance_verify_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: SCHEME=ec /root/vitastor/tests/test_rebalance_verify.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_rebalance_verify_ec_imm: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: SCHEME=ec IMMEDIATE_COMMIT=1 /root/vitastor/tests/test_rebalance_verify.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_rm: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_rm.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_snapshot: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_snapshot.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_snapshot_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: SCHEME=ec /root/vitastor/tests/test_snapshot.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_splitbrain: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_splitbrain.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_write: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_write.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_write_xor: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: SCHEME=xor /root/vitastor/tests/test_write.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_write_no_same: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_write_no_same.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_heal_pg_size_2: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: PG_SIZE=2 /root/vitastor/tests/test_heal.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_heal_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: SCHEME=ec /root/vitastor/tests/test_heal.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + diff --git a/.gitea/workflows/tests-to-yaml.pl b/.gitea/workflows/tests-to-yaml.pl new file mode 100755 index 00000000..5a2b9a1c --- /dev/null +++ b/.gitea/workflows/tests-to-yaml.pl @@ -0,0 +1,68 @@ +#!/usr/bin/perl + +use strict; + +for my $line (<>) +{ + if ($line =~ /\.\/(test_[^\.]+)/s) + { + chomp $line; + my $test_name = $1; + my $timeout = 3; + if ($test_name eq 'test_etcd_fail' || $test_name eq 'test_heal' || $test_name eq 'test_interrupted_rebalance') + { + $timeout = 10; + } + while ($line =~ /([^\s=]+)=(\S+)/gs) + { + if ($1 eq 'SCHEME' && $2 eq 'ec') + { + $test_name .= '_ec'; + } + elsif ($1 eq 'SCHEME' && $2 eq 'xor') + { + $test_name .= '_xor'; + } + elsif ($1 eq 'IMMEDIATE_COMMIT') + { + $test_name .= '_imm'; + } + else + { + $test_name .= '_'.lc($1).'_'.$2; + } + } + $line =~ s!\./test_!/root/vitastor/tests/test_!; + # Gitea CI doesn't support artifacts yet, lol + #- name: Upload results + # uses: actions/upload-artifact\@v3 + # if: always() + # with: + # name: ${test_name}_result + # path: | + # /root/vitastor/testdata + # !/root/vitastor/testdata/*.bin + # retention-days: 5 + print <<"EOF" + $test_name: + runs-on: ubuntu-latest + needs: build + container: \${{env.TEST_IMAGE}}:\${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: $timeout + run: $line + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- \$i --------" + cat \$i + echo "" + done + +EOF +; + } +} diff --git a/docker/vitastor.gpg b/docker/vitastor.gpg index f5150ed1..1ff33ecd 100644 Binary files a/docker/vitastor.gpg and b/docker/vitastor.gpg differ