osd: Test a rewrite after a delete that missed a down OSD #147

Open
koolkhel wants to merge 2 commits from koolkhel/vitastor:pg-epoch-rewrite-test into master

View File

@@ -3231,6 +3231,171 @@ void test_pg_epoch_bump_above_reported_epoch()
printf("test_pg_epoch_bump_above_reported_epoch passed\n");
}
// Answer every queued local and peer subop as healthy OSDs would: the local
// zero-length version read finds the object at read_ver (0 = absent), writes,
// deletes and stabilizes succeed. Returns the version of the last write sent
static uint64_t answer_subops(osd_test_fixture_t & f, uint64_t read_ver)
{
uint64_t written = 0;
bool more = true;
while (more)
{
more = false;
f.pump();
while (f.bs->queued.size())
{
more = true;
auto *op = f.bs->take();
if (op->opcode == BS_OP_READ)
{
assert(op->len == 0);
op->version = read_ver;
op->retval = read_ver ? 0 : -ENOENT;
}
else if (op->opcode == BS_OP_WRITE || op->opcode == BS_OP_WRITE_STABLE)
{
written = op->version;
op->retval = op->len;
}
else
{
assert(op->opcode == BS_OP_DELETE || op->opcode == BS_OP_STABLE);
op->retval = 0;
}
std::function<void(blockstore_op_t*)>(op->callback)(op);
}
// One peer op per pass: completions may send new ones
osd_op_t *op = NULL;
for (osd_num_t osd = 2; osd <= 3 && !op; osd++)
for (int opcode: { OSD_OP_SEC_WRITE, OSD_OP_SEC_DELETE, OSD_OP_SEC_STABILIZE })
if (!op)
op = f.peer_take_opt(osd, opcode);
if (op)
{
more = true;
switch (op->req.hdr.opcode)
{
case OSD_OP_SEC_WRITE:
written = op->req.sec_rw.version;
op->reply.sec_rw.version = op->req.sec_rw.version;
f.peer_complete(op, op->req.sec_rw.len);
break;
case OSD_OP_SEC_DELETE:
op->reply.sec_del.version = op->req.sec_del.version;
f.peer_complete(op, 0);
break;
case OSD_OP_SEC_STABILIZE:
f.peer_complete(op, 0);
break;
}
}
}
return written;
}
// A delete misses a down OSD, the still degraded PG re-peers, the
// object is written again, and the OSD comes back. The rewrite must get a newer
// version than the copy the down OSD kept, or that stale copy passes for current.
// A delete without tombstones (the fixture's default) doesn't report the epoch, so
// after the delete only etcd remembers epoch 7 of the deleted chain. Object Y keeps
// an epoch-6 version in the LIST: an epoch recomputed from the LIST alone lands on 7
// again and the rewrite reuses 7:1
void test_rewrite_after_partial_delete(bool ec)
{
printf("test_rewrite_after_partial_delete(%s)\n", ec ? "ec" : "replicated");
// Replicated x2 on OSDs 1,2 or EC 2+1 on OSDs 1,2,3; we are OSD 1, the primary.
// The last OSD of the set is down from the start
osd_test_fixture_t f;
osd_num_t stale_osd = ec ? 3 : 2;
uint64_t stale_ver = 0x70001;
json11::Json::array set;
for (osd_num_t osd = 1; osd <= stale_osd; osd++)
set.push_back((double)osd);
if (ec)
f.configure_ec_pool(2, 1, /*primary_osd*/ 1);
else
f.configure_replicated_pool(1, 2, 1, 1, { { 1, 2 } });
f.st_cli->set("/vitastor/pg/history/1/1", json11::Json::object {
{ "osd_sets", json11::Json::array { set } },
{ "all_peers", set },
{ "epoch", 7 },
});
f.start(json11::Json::object {
{ "osd_num", 1 },
{ "etcd_address", "127.0.0.1:2379" },
{ "immediate_commit", "all" },
{ "block_size", 131072 },
{ "bitmap_granularity", 4096 },
{ "no_recovery", true },
});
if (ec)
f.connect_peer(2);
// Object `num` as listed by `osd`: role = position in the set for EC
auto obj = [&](uint64_t num, osd_num_t osd) -> object_id
{
return { .inode = INODE_WITH_POOL(1, num), .stripe = ec ? osd-1 : 0 };
};
auto reply_lists = [&](std::function<std::vector<obj_ver_id>(osd_num_t)> list_of)
{
auto local = list_of(1);
f.reply_local_list(local, local.size());
for (osd_num_t osd = 2; osd <= stale_osd; osd++)
{
if (f.peer_take_opt(osd, OSD_OP_SEC_LIST))
{
auto l = list_of(osd);
f.reply_peer_list(osd, l, l.size());
}
}
f.pump();
};
// X at 7:1 and Y at 6:1 on the live OSDs: the PG comes up degraded
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{ { obj(1, osd), stale_ver }, { obj(2, osd), 0x60001 } }; });
assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE);
// Delete X: it reaches the live OSDs only
int del_retval = -1;
auto *del = make_self_rw(OSD_OP_DELETE, INODE_WITH_POOL(1, 1), 0, 0);
del->callback = [&](osd_op_t *op) { del_retval = op->reply.hdr.retval; };
f.exec(del);
answer_subops(f, stale_ver);
assert(del_retval == 0);
assert((del->reply.del.flags & OSD_DEL_LEFT_ON_DEAD) && del->reply.del.left_on_dead_count == 1);
assert(((uint32_t*)(&del->reply.del + 1))[0] == stale_osd);
delete del;
// The PG re-peers, still without the stale OSD: X is gone from every LIST
f.repeer_pgs(1);
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{ { obj(2, osd), 0x60001 } }; });
assert((f.pg(1, 1).state & PG_ACTIVE) && f.pg(1, 1).state != PG_ACTIVE);
// Write X again
int wr_retval = -1;
auto *wr = make_write_op(INODE_WITH_POOL(1, 1), 0, 4096, 0xcd);
wr->callback = [&](osd_op_t *op) { wr_retval = op->reply.hdr.retval; };
f.exec(wr);
uint64_t new_ver = answer_subops(f, 0);
assert(wr_retval == 4096);
delete wr;
printf("rewrite got version %jx, the stale copy has %jx\n", new_ver, stale_ver);
assert(new_ver > stale_ver);
// The stale OSD returns with X still at 7:1: X is degraded, and the stale OSD
// holds no current part of it (replicated marks the copy outdated, EC ignores it)
f.connect_peer(stale_osd);
reply_lists([&](osd_num_t osd) { return std::vector<obj_ver_id>{
{ obj(1, osd), osd == stale_osd ? stale_ver : new_ver }, { obj(2, osd), 0x60001 } }; });
pg_osd_set_state_t *st = find_object_state(f.pg(1, 1), obj(1, 1));
assert(st && (st->state & OBJ_DEGRADED));
for (auto & chunk: st->osd_set)
assert(chunk.osd_num != stale_osd || (chunk.loc_bad & LOC_OUTDATED));
printf("test_rewrite_after_partial_delete(%s) passed\n", ec ? "ec" : "replicated");
}
// A partial unstable chain left by a failed/rejected EC write becomes the committed
// version when it holds pg_data_size pieces: a flush may have stabilized some of them
// already, and the history check lets a later peering activate with pg_data_size OSDs
@@ -4304,6 +4469,8 @@ int main(int narg, char *args[])
test_peering_outdated_copy_other_role();
test_pg_epoch_base_is_reported_epoch();
test_pg_epoch_bump_above_reported_epoch();
test_rewrite_after_partial_delete(false);
test_rewrite_after_partial_delete(true);
test_chained_read_large_chain_info();
test_peering_partial_unstable_ec_rolls_back();
test_peering_flush_primary_lost({ 2, 1, 2 }, 2);