Add resize and resize-auto tests

.gitea/workflows/buildenv.Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update
 RUN apt-get -y install etcd qemu-system-x86 qemu-block-extra qemu-utils fio libasan5 \
     liburing1 liburing-dev libgoogle-perftools-dev devscripts libjerasure-dev cmake libibverbs-dev libisal-dev
 RUN apt-get -y build-dep fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
-RUN apt-get -y install jq lp-solve sudo nfs-common
+RUN apt-get update && apt-get -y install jq lp-solve sudo nfs-common fdisk parted
 RUN apt-get --download-only source fio qemu=`dpkg -s qemu-system-x86|grep ^Version:|awk '{print $2}'`
 
 RUN set -ex; \

.gitea/workflows/test.yml

@@ -828,6 +828,42 @@ jobs:
           echo ""
         done
 
+  test_resize:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_resize.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
+  test_resize_auto:
+    runs-on: ubuntu-latest
+    needs: build
+    container: ${{env.TEST_IMAGE}}:${{github.sha}}
+    steps:
+    - name: Run test
+      id: test
+      timeout-minutes: 3
+      run: /root/vitastor/tests/test_resize_auto.sh
+    - name: Print logs
+      if: always() && steps.test.outcome == 'failure'
+      run: |
+        for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
+          echo "-------- $i --------"
+          cat $i
+          echo ""
+        done
+
   test_snapshot_pool2:
     runs-on: ubuntu-latest
     needs: build

tests/run_tests.sh

@@ -68,6 +68,9 @@ TEST_NAME=csum_4k_dmj  OSD_ARGS="--data_csum_type crc32c --inmemory_metadata fal
 TEST_NAME=csum_4k_dj   OSD_ARGS="--data_csum_type crc32c --inmemory_journal false" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 TEST_NAME=csum_4k      OSD_ARGS="--data_csum_type crc32c" OFFSET_ARGS=$OSD_ARGS ./test_heal.sh
 
+./test_resize.sh
+./test_resize_auto.sh
+
 ./test_snapshot_pool2.sh
 
 ./test_osd_tags.sh

tests/test_resize.sh

@@ -3,6 +3,7 @@
 PG_COUNT=${PG_COUNT:-32}
 
 . `dirname $0`/run_3osds.sh
+check_qemu
 
 LD_PRELOAD="build/src/client/libfio_vitastor.so" \
     fio -thread -name=test -ioengine=build/src/client/libfio_vitastor.so -bs=4M -direct=1 -iodepth=4 \
@@ -28,12 +29,12 @@ for i in $(seq 1 $OSD_COUNT); do
     data_offset=$(echo $offsets | jq -r .data_offset)
     build/src/disk_tool/vitastor-disk dump-journal --json ./testdata/bin/test_osd$i.bin 4096 0 $meta_offset >./testdata/journal_before_resize.json
     build/src/disk_tool/vitastor-disk dump-meta ./testdata/bin/test_osd$i.bin 4096 $meta_offset $((data_offset-meta_offset)) >./testdata/meta_before_resize.json
-    build/src/disk_tool/vitastor-disk resize \
+    build/src/disk_tool/vitastor-disk raw-resize \
         $(build/src/disk_tool/vitastor-disk simple-offsets --format options ./testdata/bin/test_osd$i.bin 2>/dev/null) \
         --new_meta_offset 0 \
         --new_meta_len $((1024*1024)) \
         --new_journal_offset $((1024*1024)) \
-        --new_data_offset $((128*1024*1024))
+        --new_data_offset $((128*1024*1024+32768))
     build/src/disk_tool/vitastor-disk dump-journal --json ./testdata/bin/test_osd$i.bin 4096 $((1024*1024)) $((127*1024*1024)) >./testdata/journal_after_resize.json
     build/src/disk_tool/vitastor-disk dump-meta ./testdata/bin/test_osd$i.bin 4096 0 $((1024*1024)) >./testdata/meta_after_resize.json
     if ! (cat ./testdata/meta_before_resize.json ./testdata/meta_after_resize.json | \
@@ -41,7 +42,7 @@ for i in $(seq 1 $OSD_COUNT); do
         format_error "OSD $i metadata corrupted after resizing"
     fi
     if ! (cat ./testdata/journal_before_resize.json ./testdata/journal_after_resize.json | \
-        jq -e -s 'map([ .[].entries[] | del(.crc32, .crc32_prev, .valid, .loc, .start) ]) | .[0] == .[1] and (.[0] | length) > 1'); then
+        jq -e -s 'map([ .[] | del(.crc32, .crc32_prev, .valid, .loc, .start) ]) | .[0] == .[1] and (.[0] | length) > 1'); then
         format_error "OSD $i journal corrupted after resizing"
     fi
 done
@@ -53,7 +54,7 @@ for i in $(seq 1 $OSD_COUNT); do
         --data_device ./testdata/bin/test_osd$i.bin \
         --meta_offset 0 \
         --journal_offset $((1024*1024)) \
-        --data_offset $((128*1024*1024)) >>./testdata/osd$i.log 2>&1 &
+        --data_offset $((128*1024*1024+32768)) >>./testdata/osd$i.log 2>&1 &
     eval OSD${i}_PID=$!
 done
 

tests/test_resize_auto.sh

@@ -0,0 +1,93 @@
+#!/bin/bash -ex
+
+ANTIETCD=1
+. `dirname $0`/common.sh
+
+[[ -e build/src/disk_tool/vitastor-disk-test ]] || ln -s vitastor-disk build/src/disk_tool/vitastor-disk-test
+
+dd if=/dev/zero of=./testdata/bin/test_osd1.bin bs=1 count=1 seek=$((100*1024*1024*1024-1))
+LOOP1=$(sudo losetup --show -f ./testdata/bin/test_osd1.bin)
+trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1"' || true' EXIT
+dd if=/dev/zero of=./testdata/bin/test_meta.bin bs=1 count=1 seek=$((1024*1024*1024-1))
+LOOP2=$(sudo losetup --show -f ./testdata/bin/test_meta.bin)
+trap "kill -9 $(jobs -p) || true; sudo losetup -d $LOOP1 $LOOP2"' || true' EXIT
+
+# also test prepare --hybrid :)
+# non-vitastor random type UUID to prevent udev activation
+mount | grep '/dev type devtmpfs' || sudo mount udev /dev/ -t devtmpfs
+sudo build/src/disk_tool/vitastor-disk-test prepare --no_init 1 --meta_reserve 1x,1M \
+    --block_size 131072 --osd_num 987654 --part_type_uuid 0df42ae0-3695-4395-a957-7d5ff3645c56 \
+    --hybrid --fast-devices $LOOP2 $LOOP1
+
+# write almost empty journal
+node <<EOF > ./testdata/journal.json
+console.log(JSON.stringify([
+    {"type":"start","start":"0x1000"},
+    {"type":"big_write_instant","inode":"0x1000000000001","stripe":"0xc60000","ver":"10","offset":0,"len":131072,"loc":"0x18ffdc0000","bitmap":"ffffffff"}
+]));
+EOF
+sudo build/src/disk_tool/vitastor-disk write-journal ${LOOP1}p1 < ./testdata/journal.json
+sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
+diff ./testdata/j1.json ./testdata/j2.json
+
+# write fake metadata items in the end
+DATA_DEV_SIZE=$(sudo blockdev --getsize64 ${LOOP1}p1)
+node <<EOF > ./testdata/meta.json
+console.log(JSON.stringify({
+    version: "0.9",
+    meta_block_size: 4096,
+    data_block_size: 131072,
+    bitmap_granularity: 4096,
+    data_csum_type: "none",
+    csum_block_size: 0,
+    entries: [ ...new Array(100).keys() ].map(i => ({
+        block: (819183-100)+i, // 819183 = (rounded partition size-4k) / 128k
+        pool: 1,
+        inode: "0x1",
+        stripe: "0x"+Number(i*0x20000).toString(16),
+        version: 10,
+        bitmap: "ffffffff",
+        ext_bitmap: "ffffffff",
+    })),
+}));
+EOF
+
+# also test write & dump
+sudo build/src/disk_tool/vitastor-disk write-meta ${LOOP1}p1 < ./testdata/meta.json
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 > ./testdata/compare.json
+jq -S < ./testdata/meta.json > ./testdata/1.json
+jq -S < ./testdata/compare.json > ./testdata/2.json
+diff ./testdata/1.json ./testdata/2.json
+
+# move journal & meta back, data will become smaller; end indexes should be shifted by -1251
+sudo build/src/disk_tool/vitastor-disk-test resize --move-journal '' --move-meta '' ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S > ./testdata/2.json
+jq -S '. + {"entries": [ .entries[] | (. + { "block": (.block-1251) }) ]}' < ./testdata/meta.json > ./testdata/1.json
+diff ./testdata/1.json ./testdata/2.json
+sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+jq -S '[ (.[] + {"valid":true}) | (if .type == "big_write_instant" then . + {"loc":"0x18f6160000"} else . end) ]' < ./testdata/journal.json > ./testdata/j1.json
+diff ./testdata/j1.json ./testdata/j2.json
+
+# move journal & meta out, data will become larger; end indexes should be shifted back by +1251
+sudo build/src/disk_tool/vitastor-disk-test resize --move-journal ${LOOP2}p1 --move-meta ${LOOP2}p2 ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S > ./testdata/2.json
+jq -S < ./testdata/meta.json > ./testdata/1.json
+diff ./testdata/1.json ./testdata/2.json
+jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
+sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+
+# reduce data device size by exactly 128k * 99 (occupied blocks); exactly 1 should be left in place :)
+sudo build/src/disk_tool/vitastor-disk-test resize --data-size $((DATA_DEV_SIZE-128*1024*99)) ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S > ./testdata/2.json
+jq -S '. + {"entries": ([ .entries[] | (. + { "block": (.block | if . > 819183-100 then .-(819183-100+1) else 819183-100 end) }) ] | .[1:] + [ .[0] ])}' < ./testdata/meta.json > ./testdata/1.json
+diff ./testdata/1.json ./testdata/2.json
+jq -S '[ .[] + {"valid":true} ]' < ./testdata/journal.json > ./testdata/j1.json
+sudo build/src/disk_tool/vitastor-disk dump-journal --json --format data ${LOOP1}p1 | jq -S '[ .[] | del(.crc32, .crc32_prev) ]' > ./testdata/j2.json
+
+# extend data device size to maximum
+sudo build/src/disk_tool/vitastor-disk-test resize --data-size max ${LOOP1}p1
+sudo build/src/disk_tool/vitastor-disk dump-meta ${LOOP1}p1 | jq -S > ./testdata/2.json
+diff ./testdata/1.json ./testdata/2.json
+
+format_green OK