diff --git a/src/test/osd_test.cpp b/src/test/osd_test.cpp index 50a25299..c6659618 100644 --- a/src/test/osd_test.cpp +++ b/src/test/osd_test.cpp @@ -3231,6 +3231,171 @@ void test_pg_epoch_bump_above_reported_epoch() printf("test_pg_epoch_bump_above_reported_epoch passed\n"); } +// Answer every queued local and peer subop as healthy OSDs would: the local +// zero-length version read finds the object at read_ver (0 = absent), writes, +// deletes and stabilizes succeed. Returns the version of the last write sent +static uint64_t answer_subops(osd_test_fixture_t & f, uint64_t read_ver) +{ + uint64_t written = 0; + bool more = true; + while (more) + { + more = false; + f.pump(); + while (f.bs->queued.size()) + { + more = true; + auto *op = f.bs->take(); + if (op->opcode == BS_OP_READ) + { + assert(op->len == 0); + op->version = read_ver; + op->retval = read_ver ? 0 : -ENOENT; + } + else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE) + { + written = op->version; + op->retval = op->len; + } + else + { + assert(op->opcode == BS_OP_DELETE || op->opcode == BS_OP_STABLE); + op->retval = 0; + } + std::function(op->callback)(op); + } + // One peer op per pass: completions may send new ones + osd_op_t *op = NULL; + for (osd_num_t osd = 2; osd <= 3 && !op; osd++) + for (int opcode: { OSD_OP_SEC_WRITE, OSD_OP_SEC_DELETE, OSD_OP_SEC_STABILIZE }) + if (!op) + op = f.peer_take_opt(osd, opcode); + if (op) + { + more = true; + switch (op->req.hdr.opcode) + { + case OSD_OP_SEC_WRITE: + written = op->req.sec_rw.version; + op->reply.sec_rw.version = op->req.sec_rw.version; + f.peer_complete(op, op->req.sec_rw.len); + break; + case OSD_OP_SEC_DELETE: + op->reply.sec_del.version = op->req.sec_del.version; + f.peer_complete(op, 0); + break; + case OSD_OP_SEC_STABILIZE: + f.peer_complete(op, 0); + break; + } + } + } + return written; +} + +// A delete misses a down OSD, the still degraded PG re-peers, the +// object is written again, and the OSD comes back. The rewrite must get a newer +// version than the copy the down OSD kept, or that stale copy passes for current. +// A delete without tombstones (the fixture's default) doesn't report the epoch, so +// after the delete only etcd remembers epoch 7 of the deleted chain. Object Y keeps +// an epoch-6 version in the LIST: an epoch recomputed from the LIST alone lands on 7 +// again and the rewrite reuses 7:1 +void test_rewrite_after_partial_delete(bool ec) +{ + printf("test_rewrite_after_partial_delete(%s)\n", ec ? "ec" : "replicated"); + + // Replicated x2 on OSDs 1,2 or EC 2+1 on OSDs 1,2,3; we are OSD 1, the primary. + // The last OSD of the set is down from the start + osd_test_fixture_t f; + osd_num_t stale_osd = ec ? 3 : 2; + uint64_t stale_ver = 0x70001; + json11::Json::array set; + for (osd_num_t osd = 1; osd <= stale_osd; osd++) + set.push_back((double)osd); + if (ec) + f.configure_ec_pool(2, 1, /*primary_osd*/ 1); + else + f.configure_replicated_pool(1, 2, 1, 1, { { 1, 2 } }); + f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object { + { "osd_sets", json11::Json::array { set } }, + { "all_peers", set }, + { "epoch", 7 }, + }); + f.start(json11::Json::object { + { "osd_num", 1 }, + { "etcd_address", "127.0.0.1:2379" }, + { "immediate_commit", "all" }, + { "block_size", 131072 }, + { "bitmap_granularity", 4096 }, + { "no_recovery", true }, + }); + if (ec) + f.connect_peer(2); + + // Object `num` as listed by `osd`: role = position in the set for EC + auto obj = [&](uint64_t num, osd_num_t osd) -> object_id + { + return { .inode = INODE_WITH_POOL(1, num), .stripe = ec ? osd-1 : 0 }; + }; + auto reply_lists = [&](std::function(osd_num_t)> list_of) + { + auto local = list_of(1); + f.reply_local_list(local, local.size()); + for (osd_num_t osd = 2; osd <= stale_osd; osd++) + { + if (f.peer_take_opt(osd, OSD_OP_SEC_LIST)) + { + auto l = list_of(osd); + f.reply_peer_list(osd, l, l.size()); + } + } + f.pump(); + }; + + // X at 7:1 and Y at 6:1 on the live OSDs: the PG comes up degraded + reply_lists([&](osd_num_t osd) { return std::vector{ { obj(1, osd), stale_ver }, { obj(2, osd), 0x60001 } }; }); + assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE); + + // Delete X: it reaches the live OSDs only + int del_retval = -1; + auto *del = make_self_rw(OSD_OP_DELETE, INODE_WITH_POOL(1, 1), 0, 0); + del->callback = [&](osd_op_t *op) { del_retval = op->reply.hdr.retval; }; + f.exec(del); + answer_subops(f, stale_ver); + assert(del_retval == 0); + assert((del->reply.del.flags & OSD_DEL_LEFT_ON_DEAD) && del->reply.del.left_on_dead_count == 1); + assert(((uint32_t*)(&del->reply.del + 1))[0] == stale_osd); + delete del; + + // The PG re-peers, still without the stale OSD: X is gone from every LIST + f.repeer_pgs(1); + reply_lists([&](osd_num_t osd) { return std::vector{ { obj(2, osd), 0x60001 } }; }); + assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE); + + // Write X again + int wr_retval = -1; + auto *wr = make_write_op(INODE_WITH_POOL(1, 1), 0, 4096, 0xcd); + wr->callback = [&](osd_op_t *op) { wr_retval = op->reply.hdr.retval; }; + f.exec(wr); + uint64_t new_ver = answer_subops(f, 0); + assert(wr_retval == 4096); + delete wr; + printf("rewrite got version %jx, the stale copy has %jx\n", new_ver, stale_ver); + assert(new_ver > stale_ver); + + // The stale OSD returns with X still at 7:1: X is degraded, and the stale OSD + // holds no current part of it (replicated marks the copy outdated, EC ignores it) + f.connect_peer(stale_osd); + reply_lists([&](osd_num_t osd) { return std::vector{ + { obj(1, osd), osd == stale_osd ? stale_ver : new_ver }, { obj(2, osd), 0x60001 } }; }); + pg_osd_set_state_t *st = find_object_state(f.pg(1, 1), obj(1, 1)); + assert(st && (st->state & OBJ_DEGRADED)); + for (auto & chunk: st->osd_set) + assert(chunk.osd_num != stale_osd || (chunk.loc_bad & LOC_OUTDATED)); + + printf("test_rewrite_after_partial_delete(%s) passed\n", ec ? "ec" : "replicated"); +} + // A partial unstable chain left by a failed/rejected EC write becomes the committed // version when it holds pg_data_size pieces: a flush may have stabilized some of them // already, and the history check lets a later peering activate with pg_data_size OSDs @@ -4304,6 +4469,8 @@ int main(int narg, char *args[]) test_peering_outdated_copy_other_role(); test_pg_epoch_base_is_reported_epoch(); test_pg_epoch_bump_above_reported_epoch(); + test_rewrite_after_partial_delete(false); + test_rewrite_after_partial_delete(true); test_chained_read_large_chain_info(); test_peering_partial_unstable_ec_rolls_back(); test_peering_flush_primary_lost({ 2, 1, 2 }, 2);