Compare commits

...

1 Commits

Author SHA1 Message Date
Vitaliy Filippov ee1dcfea25 Test bug similar to 307c1731c1
Test / test_change_pg_size (push) Successful in 10s Details
Test / test_create_nomaxid (push) Successful in 8s Details
Test / test_etcd_fail (push) Successful in 49s Details
Test / test_failure_domain (push) Successful in 9s Details
Test / test_interrupted_rebalance (push) Successful in 1m9s Details
Test / test_interrupted_rebalance_imm (push) Successful in 1m1s Details
Test / test_interrupted_rebalance_ec (push) Successful in 1m5s Details
Test / test_interrupted_rebalance_ec_imm (push) Successful in 56s Details
Test / test_minsize_1 (push) Successful in 15s Details
Test / test_move_reappear (push) Successful in 17s Details
Test / test_rebalance_verify (push) Successful in 1m44s Details
Test / test_rebalance_verify_imm (push) Successful in 1m44s Details
Test / test_rebalance_verify_ec (push) Successful in 1m56s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 1m51s Details
Test / test_rm (push) Successful in 12s Details
Test / test_snapshot (push) Successful in 16s Details
Test / test_snapshot_ec (push) Successful in 19s Details
Test / test_splitbrain (push) Successful in 14s Details
Test / test_write (push) Successful in 50s Details
Test / test_write_xor (push) Successful in 1m0s Details
Test / test_write_no_same (push) Successful in 10s Details
Test / test_heal_pg_size_2 (push) Successful in 2m26s Details
Test / test_heal_ec (push) Successful in 2m22s Details
Test / test_heal_imm_ec (push) Successful in 3m26s Details
Test / test_scrub (push) Successful in 24s Details
Test / test_scrub_zero_osd_2 (push) Successful in 40s Details
Test / test_scrub_xor (push) Successful in 39s Details
Test / test_scrub_pg_size_3 (push) Successful in 21s Details
Test / test_scrub_pg_size_6_pg_minsize_4_osd_count_6_ec (push) Successful in 29s Details
Test / test_scrub_ec (push) Successful in 17s Details
2023-06-02 17:36:56 +03:00
5 changed files with 60 additions and 32 deletions

View File

@ -550,6 +550,24 @@ jobs:
echo ""
done
test_heal_imm_ec:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 10
run: IMMEDIATE_COMMIT=1 SCHEME=ec /root/vitastor/tests/test_heal.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_scrub:
runs-on: ubuntu-latest
needs: build

View File

@ -536,14 +536,27 @@ resume_1:
return false;
}
// zero out old metadata entry
{
clean_disk_entry *old_entry = (clean_disk_entry*)((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size);
if (old_entry->oid.inode != 0 && old_entry->oid != cur.oid)
{
printf("Fatal error (metadata corruption or bug): tried to wipe metadata entry %lu (%lx:%lx v%lu) as old location of %lx:%lx\n",
old_clean_loc >> bs->dsk.block_order, old_entry->oid.inode, old_entry->oid.stripe,
old_entry->version, cur.oid.inode, cur.oid.stripe);
exit(1);
}
}
memset((uint8_t*)meta_old.buf + meta_old.pos*bs->dsk.clean_entry_size, 0, bs->dsk.clean_entry_size);
await_sqe(15);
data->iov = (struct iovec){ meta_old.buf, bs->dsk.meta_block_size };
data->callback = simple_callback_w;
my_uring_prep_writev(
sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_old.sector
);
wait_count++;
if (meta_old.sector != meta_new.sector)
{
await_sqe(15);
data->iov = (struct iovec){ meta_old.buf, bs->dsk.meta_block_size };
data->callback = simple_callback_w;
my_uring_prep_writev(
sqe, bs->dsk.meta_fd, &data->iov, 1, bs->dsk.meta_offset + bs->dsk.meta_block_size + meta_old.sector
);
wait_count++;
}
}
if (has_delete)
{

View File

@ -790,7 +790,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
unstab = unstab < ov.version ? ov.version : unstab;
if (je->type == JE_SMALL_WRITE_INSTANT)
{
bs->mark_stable(ov, true);
bs->mark_stable(ov);
}
}
}
@ -890,7 +890,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
unstab = unstab < ov.version ? ov.version : unstab;
if (je->type == JE_BIG_WRITE_INSTANT)
{
bs->mark_stable(ov, true);
bs->mark_stable(ov);
}
}
}
@ -904,7 +904,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
.oid = je->stable.oid,
.version = je->stable.version,
};
bs->mark_stable(ov, true);
bs->mark_stable(ov);
}
else if (je->type == JE_ROLLBACK)
{
@ -961,7 +961,7 @@ int blockstore_init_journal::handle_journal_part(void *buf, uint64_t done_pos, u
bs->journal.used_sectors[proc_pos]++;
// Deletions are treated as immediately stable, because
// "2-phase commit" (write->stabilize) isn't sufficient for them anyway
bs->mark_stable(ov, true);
bs->mark_stable(ov);
}
// Ignore delete if neither preceding dirty entries nor the clean one are present
}

View File

@ -46,6 +46,7 @@ SCHEME=xor ./test_write.sh
PG_SIZE=2 ./test_heal.sh
SCHEME=ec ./test_heal.sh
IMMEDIATE_COMMIT=1 SCHEME=ec ./test_heal.sh
./test_scrub.sh
ZERO_OSD=2 ./test_scrub.sh

View File

@ -12,7 +12,7 @@ PG_COUNT=32
. `dirname $0`/run_3osds.sh
check_qemu
IMG_SIZE=960
IMG_SIZE=128
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
@ -22,32 +22,28 @@ LD_PRELOAD="build/src/libfio_vitastor.so" \
kill_osds()
{
sleep 5
kill -9 $OSD1_PID
$ETCDCTL del /vitastor/osd/state/1
for i in 2 3 4 5 6 7; do
sleep 15
echo Killing OSD $i and starting OSD $((i-1))
p=OSD${i}_PID
kill -9 ${!p}
$ETCDCTL del /vitastor/osd/state/$i
start_osd $((i-1))
sleep 15
for i in {1..60}; do
sleep 5
K=$((1 + RANDOM % OSD_COUNT))
eval "kill -9 \$OSD${K}_PID"
$ETCDCTL del /vitastor/osd/state/$K
start_osd $K
done
sleep 5
start_osd 7
sleep 5
}
kill_osds &
LD_PRELOAD="build/src/libfio_vitastor.so" \
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \
-refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \
-refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120 &>/dev/null &
LD_PRELOAD="build/src/libfio_vitastor.so" \
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \
-refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120 &>/dev/null &
LD_PRELOAD="build/src/libfio_vitastor.so" \
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=512k -direct=1 -iodepth=8 -fsync=256 -rw=write \
-refill_buffers=1 -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=1000 -runtime=120
qemu-img convert -S 4096 -p \
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \