diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ea7650ae..8c05d978 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -550,6 +550,24 @@ jobs: echo "" done + test_heal_imm_ec: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 10 + run: IMMEDIATE_COMMIT=1 SCHEME=ec /root/vitastor/tests/test_heal.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/blockstore_flush.cpp b/src/blockstore_flush.cpp index 46e79fc9..53489c2e 100644 --- a/src/blockstore_flush.cpp +++ b/src/blockstore_flush.cpp @@ -536,14 +536,27 @@ resume_1: return false; } // zero out old metadata entry + { + clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size); + if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid) + { + printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %lu (%lx:%lx v%lu) as old location of %lx:%lx\n", + old_clean_loc >> bs->dsk.block_order, old_entry->oid.inode, old_entry->oid.stripe, + old_entry->version, cur.oid.inode, cur.oid.stripe); + exit(1); + } + } memset((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size); - await_sqe(15); - data->iov = (struct iovec){ meta_old.buf, bs->dsk.meta_block_size }; - data->callback = simple_callback_w; - my_uring_prep_writev( - sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_old.sector - ); - wait_count++; + if (meta_old.sector != meta_new.sector) + { + await_sqe(15); + data->iov = (struct iovec){ meta_old.buf, bs->dsk.meta_block_size }; + data->callback = simple_callback_w; + my_uring_prep_writev( + sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_old.sector + ); + wait_count++; + } } if (has_delete) { diff --git a/src/blockstore_init.cpp b/src/blockstore_init.cpp index 9453ff69..e303123f 100644 --- a/src/blockstore_init.cpp +++ b/src/blockstore_init.cpp @@ -790,7 +790,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u unstab = unstab < ov.version ? ov.version : unstab; if (je->type == JE_SMALL_WRITE_INSTANT) { - bs->mark_stable(ov, true); + bs->mark_stable(ov); } } } @@ -890,7 +890,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u unstab = unstab < ov.version ? ov.version : unstab; if (je->type == JE_BIG_WRITE_INSTANT) { - bs->mark_stable(ov, true); + bs->mark_stable(ov); } } } @@ -904,7 +904,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u .oid = je->stable.oid, .version = je->stable.version, }; - bs->mark_stable(ov, true); + bs->mark_stable(ov); } else if (je->type == JE_ROLLBACK) { @@ -961,7 +961,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u bs->journal.used_sectors[proc_pos]++; // Deletions are treated as immediately stable, because // "2-phase commit" (write->stabilize) isn't sufficient for them anyway - bs->mark_stable(ov, true); + bs->mark_stable(ov); } // Ignore delete if neither preceding dirty entries nor the clean one are present } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index dc0799e5..a4983d4b 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -46,6 +46,7 @@ SCHEME=xor ./test_write.sh PG_SIZE=2 ./test_heal.sh SCHEME=ec ./test_heal.sh +IMMEDIATE_COMMIT=1 SCHEME=ec ./test_heal.sh ./test_scrub.sh ZERO_OSD=2 ./test_scrub.sh diff --git a/tests/test_heal.sh b/tests/test_heal.sh index c5308751..6b95bd1d 100755 --- a/tests/test_heal.sh +++ b/tests/test_heal.sh @@ -12,7 +12,7 @@ PG_COUNT=32 . `dirname $0`/run_3osds.sh check_qemu -IMG_SIZE=960 +IMG_SIZE=128 $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}' @@ -22,32 +22,28 @@ LD_PRELOAD="build/src/libfio_vitastor.so" \ kill_osds() { - sleep 5 - - kill -9 $OSD1_PID - $ETCDCTL del /vitastor/osd/state/1 - - for i in 2 3 4 5 6 7; do - sleep 15 - echo Killing OSD $i and starting OSD $((i-1)) - p=OSD${i}_PID - kill -9 ${!p} - $ETCDCTL del /vitastor/osd/state/$i - start_osd $((i-1)) - sleep 15 + for i in {1..60}; do + sleep 5 + K=$((1 + RANDOM % OSD_COUNT)) + eval "kill -9 \$OSD${K}_PID" + $ETCDCTL del /vitastor/osd/state/$K + start_osd $K done - - sleep 5 - start_osd 7 - - sleep 5 } kill_osds & LD_PRELOAD="build/src/libfio_vitastor.so" \ - fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \ - -refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \ + -refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120 &>/dev/null & + +LD_PRELOAD="build/src/libfio_vitastor.so" \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \ + -refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120 &>/dev/null & + +LD_PRELOAD="build/src/libfio_vitastor.so" \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \ + -refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120 qemu-img convert -S 4096 -p \ -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \