osd: Test a rewrite after a delete that missed a down OSD #147
@@ -3231,6 +3231,171 @@ void test_pg_epoch_bump_above_reported_epoch()
|
||||
printf("test_pg_epoch_bump_above_reported_epoch passed\n");
|
||||
}
|
||||
|
||||
// Answer every queued local and peer subop as healthy OSDs would: the local
|
||||
// zero-length version read finds the object at read_ver (0 = absent), writes,
|
||||
// deletes and stabilizes succeed. Returns the version of the last write sent
|
||||
static uint64_t answer_subops(osd_test_fixture_t & f, uint64_t read_ver)
|
||||
{
|
||||
uint64_t written = 0;
|
||||
bool more = true;
|
||||
while (more)
|
||||
{
|
||||
more = false;
|
||||
f.pump();
|
||||
while (f.bs->queued.size())
|
||||
{
|
||||
more = true;
|
||||
auto *op = f.bs->take();
|
||||
if (op->opcode == BS_OP_READ)
|
||||
{
|
||||
assert(op->len == 0);
|
||||
op->version = read_ver;
|
||||
op->retval = read_ver ? 0 : -ENOENT;
|
||||
}
|
||||
else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE)
|
||||
{
|
||||
written = op->version;
|
||||
op->retval = op->len;
|
||||
}
|
||||
else
|
||||
{
|
||||
assert(op->opcode == BS_OP_DELETE || op->opcode == BS_OP_STABLE);
|
||||
op->retval = 0;
|
||||
}
|
||||
std::function<void(blockstore_op_t*)>(op->callback)(op);
|
||||
}
|
||||
// One peer op per pass: completions may send new ones
|
||||
osd_op_t *op = NULL;
|
||||
for (osd_num_t osd = 2; osd <= 3 && !op; osd++)
|
||||
for (int opcode: { OSD_OP_SEC_WRITE, OSD_OP_SEC_DELETE, OSD_OP_SEC_STABILIZE })
|
||||
if (!op)
|
||||
op = f.peer_take_opt(osd, opcode);
|
||||
if (op)
|
||||
{
|
||||
more = true;
|
||||
switch (op->req.hdr.opcode)
|
||||
{
|
||||
case OSD_OP_SEC_WRITE:
|
||||
written = op->req.sec_rw.version;
|
||||
op->reply.sec_rw.version = op->req.sec_rw.version;
|
||||
f.peer_complete(op, op->req.sec_rw.len);
|
||||
break;
|
||||
case OSD_OP_SEC_DELETE:
|
||||
op->reply.sec_del.version = op->req.sec_del.version;
|
||||
f.peer_complete(op, 0);
|
||||
break;
|
||||
case OSD_OP_SEC_STABILIZE:
|
||||
f.peer_complete(op, 0);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return written;
|
||||
}
|
||||
|
||||
// A delete misses a down OSD, the still degraded PG re-peers, the
|
||||
// object is written again, and the OSD comes back. The rewrite must get a newer
|
||||
// version than the copy the down OSD kept, or that stale copy passes for current.
|
||||
// A delete without tombstones (the fixture's default) doesn't report the epoch, so
|
||||
// after the delete only etcd remembers epoch 7 of the deleted chain. Object Y keeps
|
||||
// an epoch-6 version in the LIST: an epoch recomputed from the LIST alone lands on 7
|
||||
// again and the rewrite reuses 7:1
|
||||
void test_rewrite_after_partial_delete(bool ec)
|
||||
{
|
||||
printf("test_rewrite_after_partial_delete(%s)\n", ec ? "ec" : "replicated");
|
||||
|
||||
// Replicated x2 on OSDs 1,2 or EC 2+1 on OSDs 1,2,3; we are OSD 1, the primary.
|
||||
// The last OSD of the set is down from the start
|
||||
osd_test_fixture_t f;
|
||||
osd_num_t stale_osd = ec ? 3 : 2;
|
||||
uint64_t stale_ver = 0x70001;
|
||||
json11::Json::array set;
|
||||
for (osd_num_t osd = 1; osd <= stale_osd; osd++)
|
||||
set.push_back((double)osd);
|
||||
if (ec)
|
||||
f.configure_ec_pool(2, 1, /*primary_osd*/ 1);
|
||||
else
|
||||
f.configure_replicated_pool(1, 2, 1, 1, { { 1, 2 } });
|
||||
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
|
||||
{ "osd_sets", json11::Json::array { set } },
|
||||
{ "all_peers", set },
|
||||
{ "epoch", 7 },
|
||||
});
|
||||
f.start(json11::Json::object {
|
||||
{ "osd_num", 1 },
|
||||
{ "etcd_address", "127.0.0.1:2379" },
|
||||
{ "immediate_commit", "all" },
|
||||
{ "block_size", 131072 },
|
||||
{ "bitmap_granularity", 4096 },
|
||||
{ "no_recovery", true },
|
||||
});
|
||||
if (ec)
|
||||
f.connect_peer(2);
|
||||
|
||||
// Object `num` as listed by `osd`: role = position in the set for EC
|
||||
auto obj = [&](uint64_t num, osd_num_t osd) -> object_id
|
||||
{
|
||||
return { .inode = INODE_WITH_POOL(1, num), .stripe = ec ? osd-1 : 0 };
|
||||
};
|
||||
auto reply_lists = [&](std::function<std::vector<obj_ver_id>(osd_num_t)> list_of)
|
||||
{
|
||||
auto local = list_of(1);
|
||||
f.reply_local_list(local, local.size());
|
||||
for (osd_num_t osd = 2; osd <= stale_osd; osd++)
|
||||
{
|
||||
if (f.peer_take_opt(osd, OSD_OP_SEC_LIST))
|
||||
{
|
||||
auto l = list_of(osd);
|
||||
f.reply_peer_list(osd, l, l.size());
|
||||
}
|
||||
}
|
||||
f.pump();
|
||||
};
|
||||
|
||||
// X at 7:1 and Y at 6:1 on the live OSDs: the PG comes up degraded
|
||||
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{ { obj(1, osd), stale_ver }, { obj(2, osd), 0x60001 } }; });
|
||||
assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE);
|
||||
|
||||
// Delete X: it reaches the live OSDs only
|
||||
int del_retval = -1;
|
||||
auto *del = make_self_rw(OSD_OP_DELETE, INODE_WITH_POOL(1, 1), 0, 0);
|
||||
del->callback = [&](osd_op_t *op) { del_retval = op->reply.hdr.retval; };
|
||||
f.exec(del);
|
||||
answer_subops(f, stale_ver);
|
||||
assert(del_retval == 0);
|
||||
assert((del->reply.del.flags & OSD_DEL_LEFT_ON_DEAD) && del->reply.del.left_on_dead_count == 1);
|
||||
assert(((uint32_t*)(&del->reply.del + 1))[0] == stale_osd);
|
||||
delete del;
|
||||
|
||||
// The PG re-peers, still without the stale OSD: X is gone from every LIST
|
||||
f.repeer_pgs(1);
|
||||
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{ { obj(2, osd), 0x60001 } }; });
|
||||
assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE);
|
||||
|
||||
// Write X again
|
||||
int wr_retval = -1;
|
||||
auto *wr = make_write_op(INODE_WITH_POOL(1, 1), 0, 4096, 0xcd);
|
||||
wr->callback = [&](osd_op_t *op) { wr_retval = op->reply.hdr.retval; };
|
||||
f.exec(wr);
|
||||
uint64_t new_ver = answer_subops(f, 0);
|
||||
assert(wr_retval == 4096);
|
||||
delete wr;
|
||||
printf("rewrite got version %jx, the stale copy has %jx\n", new_ver, stale_ver);
|
||||
assert(new_ver > stale_ver);
|
||||
|
||||
// The stale OSD returns with X still at 7:1: X is degraded, and the stale OSD
|
||||
// holds no current part of it (replicated marks the copy outdated, EC ignores it)
|
||||
f.connect_peer(stale_osd);
|
||||
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{
|
||||
{ obj(1, osd), osd == stale_osd ? stale_ver : new_ver }, { obj(2, osd), 0x60001 } }; });
|
||||
pg_osd_set_state_t *st = find_object_state(f.pg(1, 1), obj(1, 1));
|
||||
assert(st && (st->state & OBJ_DEGRADED));
|
||||
for (auto & chunk: st->osd_set)
|
||||
assert(chunk.osd_num != stale_osd || (chunk.loc_bad & LOC_OUTDATED));
|
||||
|
||||
printf("test_rewrite_after_partial_delete(%s) passed\n", ec ? "ec" : "replicated");
|
||||
}
|
||||
|
||||
// A partial unstable chain left by a failed/rejected EC write becomes the committed
|
||||
// version when it holds pg_data_size pieces: a flush may have stabilized some of them
|
||||
// already, and the history check lets a later peering activate with pg_data_size OSDs
|
||||
@@ -4304,6 +4469,8 @@ int main(int narg, char *args[])
|
||||
test_peering_outdated_copy_other_role();
|
||||
test_pg_epoch_base_is_reported_epoch();
|
||||
test_pg_epoch_bump_above_reported_epoch();
|
||||
test_rewrite_after_partial_delete(false);
|
||||
test_rewrite_after_partial_delete(true);
|
||||
test_chained_read_large_chain_info();
|
||||
test_peering_partial_unstable_ec_rolls_back();
|
||||
test_peering_flush_primary_lost({ 2, 1, 2 }, 2);
|
||||
|
||||
Reference in New Issue
Block a user