From 3363bc75ae4c6ef10e8412f4fe52d0d232f16901 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Wed, 18 Nov 2026 12:00:52 +0300 Subject: [PATCH] Store v1: Don't let a sync of an older big write unblock later small writes A small write following an unsynced big write waits in WAIT_BIG until that big write is synced, because a big write only gets its journal entry from the sync. ack_sync() unblocked every WAIT_BIG write of the object after any synced big write, including the ones waiting for a newer big write which that sync didn't cover. Their journal entries could then reach the disk before the newer big write's entry, and after a crash the replay put them on top of the older big write, giving a version that was never written. Stop unblocking at the next big write of the object: its own sync unblocks the small writes after it. Found by test_blockstore_chaos --seed 133 --config 17 --- src/blockstore/v1/sync.cpp | 5 ++ src/test/test_blockstore_v1.cpp | 134 ++++++++++++++++++++++++++++++++ 2 files changed, 139 insertions(+) diff --git a/src/blockstore/v1/sync.cpp b/src/blockstore/v1/sync.cpp index 1f8fd573..6805afc2 100644 --- a/src/blockstore/v1/sync.cpp +++ b/src/blockstore/v1/sync.cpp @@ -224,6 +224,11 @@ void blockstore_impl_t::ack_sync(blockstore_op_t *op) { stable_ver = dirty_it->first.version; } + if (IS_BIG_WRITE(dirty_it->second.state)) + { + // Small writes after the next big write wait for that one + is_big = false; + } dirty_it++; } if (stable_ver != 0) diff --git a/src/test/test_blockstore_v1.cpp b/src/test/test_blockstore_v1.cpp index b3c71cc8..e0533340 100644 --- a/src/test/test_blockstore_v1.cpp +++ b/src/test/test_blockstore_v1.cpp @@ -2888,6 +2888,139 @@ static void test_zero_write_over_rollback_to_zero() free(buf); } +// A small write following an unsynced big write waits until that big write is synced, because +// the big write only gets its journal entry from the sync. A sync of an EARLIER big write of the +// same object must not let it go: its entry could then reach the disk without the entry of the +// big write it depends on, and the replay would put it on top of the older big write, giving a +// version that was never written. Found by test_blockstore_chaos --seed 133 --config 17 +static void test_small_write_waits_for_its_own_big_write() +{ + printf("\n-- test_small_write_waits_for_its_own_big_write\n"); + + bs_test_t test; + test.default_cfg(); + test.config["disable_data_fsync"] = "0"; + test.config["immediate_commit"] = "none"; + test.init(); + printf("blockstore initialized\n"); + + object_id oid = { .inode = 1, .stripe = 0 }; + const uint32_t bs_size = 128*1024; + uint8_t *buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, bs_size); + + printf("write v1 (big)\n"); + blockstore_op_t op; + op.opcode = BS_OP_WRITE; + op.oid = oid; + op.version = 1; + op.offset = 0; + op.len = bs_size; + op.buf = buf; + memset(buf, 0xAA, bs_size); + test.exec_op(&op); + assert(op.retval == (int)op.len); + + // Hold the data fsync of the first sync so that it stays in flight with only v1 taken over, + // and swallow the data fsync of the sync submitted after it, so that v2 never gets journaled + ring_data_t *held_fsync = NULL; + bool sync1_done = false, swallowed = false; + test.sqe_handler = [&](io_uring_sqe *sqe) + { + if (sqe->fd != MOCK_DATA_FD || sqe->opcode != IORING_OP_FSYNC) + return false; + if (!held_fsync) + { + bool ok = test.data_disk->submit(sqe); + assert(ok); + held_fsync = (ring_data_t*)sqe->user_data; + return true; + } + if (sync1_done && !swallowed) + { + swallowed = true; + return true; + } + return false; + }; + printf("submitting the first sync and holding it in flight\n"); + blockstore_op_t sync1; + sync1.opcode = BS_OP_SYNC; + sync1.buf = NULL; + sync1.callback = [&](blockstore_op_t *op) { sync1_done = true; }; + test.bs->enqueue_op(&sync1); + for (int i = 0; i < 20 && !held_fsync; i++) + test.ringloop->loop(); + assert(held_fsync); + + printf("write v2 (big) while the sync is in flight\n"); + memset(buf, 0xBB, bs_size); + op.version = 2; + test.exec_op(&op); + assert(op.retval == (int)op.len); + + printf("write v3 (small) - it has to wait for v2 to be synced\n"); + uint8_t *buf3 = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, 4096); + memset(buf3, 0xCC, 4096); + blockstore_op_t op3; + bool op3_done = false; + op3.opcode = BS_OP_WRITE; + op3.oid = oid; + op3.version = 3; + op3.offset = 0; + op3.len = 4096; + op3.buf = buf3; + op3.callback = [&](blockstore_op_t *op) { op3_done = true; }; + test.bs->enqueue_op(&op3); + for (int i = 0; i < 20; i++) + test.ringloop->loop(); + assert(!op3_done); + + printf("releasing the first sync - it syncs v1 only\n"); + test.ringloop->mark_completed(held_fsync); + while (!sync1_done) + test.ringloop->loop(); + assert(sync1.retval == 0); + for (int i = 0; i < 20; i++) + test.ringloop->loop(); + assert(swallowed); + + printf("stabilize v1, which writes the current journal sector out\n"); + blockstore_op_t st; + st.opcode = BS_OP_STABLE; + st.len = 1; + st.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, sizeof(obj_ver_id)); + *((obj_ver_id*)st.buf) = (obj_ver_id){ .oid = oid, .version = 1 }; + test.exec_op(&st); + assert(st.retval == 0); + + printf("restarting as if the process had died\n"); + test.sqe_handler = NULL; + test.destroy_bs(); + test.ringloop->reset(); + test.init(); + + printf("the object must come back as v1, v2 or v3 - not as v3 written over v1\n"); + blockstore_op_t rd; + rd.opcode = BS_OP_READ; + rd.oid = oid; + rd.version = UINT64_MAX; + rd.offset = 0; + rd.len = bs_size; + rd.buf = (uint8_t*)memalign_or_die(MEM_ALIGNMENT, rd.len); + test.exec_op(&rd); + assert(rd.retval == (int)rd.len); + printf("got v%ju\n", rd.version); + bool is_v1 = memcheck(rd.buf, 0xAA, bs_size); + bool is_v2 = memcheck(rd.buf, 0xBB, bs_size); + bool is_v3 = memcheck(rd.buf, 0xCC, 4096) && memcheck(rd.buf+4096, 0xBB, bs_size-4096); + assert(is_v1 || is_v2 || is_v3); + + free(rd.buf); + free(st.buf); + free(buf3); + free(buf); +} + int main(int narg, char *args[]) { test_preserve_corruption(); @@ -2921,5 +3054,6 @@ int main(int narg, char *args[]) test_rollback_over_stable_write(); test_sync_waits_for_delete_taking_over_unsynced(); test_zero_write_over_rollback_to_zero(); + test_small_write_waits_for_its_own_big_write(); return 0; } -- 2.47.3