|
|
|
|
@@ -3107,16 +3107,14 @@ void test_pg_epoch_bump_above_reported_epoch()
|
|
|
|
|
printf("test_pg_epoch_bump_above_reported_epoch passed\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A partial unstable chain left by a failed/rejected EC write must not become the
|
|
|
|
|
// committed version even when it holds pg_data_size pieces: that would turn a
|
|
|
|
|
// possibly-nonexistent write into committed data with zero redundancy. Peering must
|
|
|
|
|
// fall back to the older stable version and roll the fragments back. A stable
|
|
|
|
|
// fragment, on the other hand, is possibly-acked - the OSD can never know otherwise
|
|
|
|
|
// - so it stays the target and is never rolled back, incomplete or not.
|
|
|
|
|
// A partial chain of pg_data_size pieces is not committed either, even when a
|
|
|
|
|
// degraded set holds a fragment per member: an EC 2+2 pair on the two data roles
|
|
|
|
|
// of a 2-member set still cannot decode (subcase D: refused-write fragments of a
|
|
|
|
|
// fresh object - peering must delete them, not commit).
|
|
|
|
|
// A partial unstable chain left by a failed/rejected EC write becomes the committed
|
|
|
|
|
// version when it holds pg_data_size pieces: a flush may have stabilized some of them
|
|
|
|
|
// already, and the history check lets a later peering activate with pg_data_size OSDs
|
|
|
|
|
// of a set, so a higher threshold would roll back a version stable on missing OSDs.
|
|
|
|
|
// The holders of the chain go to the PG history instead, so a peering without
|
|
|
|
|
// pg_data_size of them doesn't activate. A stable fragment is possibly-acked - the OSD
|
|
|
|
|
// can never know otherwise - so it stays the target and is never rolled back,
|
|
|
|
|
// incomplete or not.
|
|
|
|
|
//
|
|
|
|
|
// Layout: EC 2+2, PG 1/1, cur_set = role 0..3 => OSD 1..4. Object INODE_WITH_POOL(1,1)
|
|
|
|
|
// is stable at v1 on all roles; a rejected write left v2 unstable on role 0 (OSD 1)
|
|
|
|
|
@@ -3157,7 +3155,7 @@ void test_peering_partial_unstable_ec_rolls_back()
|
|
|
|
|
};
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// ============ A: partial unstable v2 falls back to stable v1 ============
|
|
|
|
|
// ============ A: partial unstable v2 on pg_data_size roles is committed ============
|
|
|
|
|
|
|
|
|
|
pg_t *pg = make_pg();
|
|
|
|
|
add_list(pg, 1, { { 0, 1 }, { 0, 2 } }, 1);
|
|
|
|
|
@@ -3166,18 +3164,21 @@ void test_peering_partial_unstable_ec_rolls_back()
|
|
|
|
|
add_list(pg, 4, { { 3, 1 } }, 1);
|
|
|
|
|
pg->calc_object_states(0);
|
|
|
|
|
|
|
|
|
|
// v1 is the target: the object is clean at v1, reads see v1, fragments roll back
|
|
|
|
|
assert(pg->total_count == 1 && pg->clean_count == 1);
|
|
|
|
|
assert(!(pg->state & (PG_HAS_INCOMPLETE|PG_HAS_DEGRADED)));
|
|
|
|
|
// v2 is the target: the object is degraded at v2, the fragments are stabilized, and their
|
|
|
|
|
// holders are in the history, so losing one before the PG is clean keeps the PG inactive
|
|
|
|
|
assert(pg->total_count == 1 && pg->clean_count == 0);
|
|
|
|
|
assert(pg->state & PG_HAS_DEGRADED);
|
|
|
|
|
assert(!(pg->state & PG_HAS_INCOMPLETE));
|
|
|
|
|
assert(pg->state & PG_HAS_UNCLEAN);
|
|
|
|
|
assert(!find_object_state(*pg, oid));
|
|
|
|
|
assert(pg->ver_override.count(oid) && pg->ver_override[oid] == 1);
|
|
|
|
|
assert(find_object_state(*pg, oid));
|
|
|
|
|
assert(!pg->ver_override.count(oid));
|
|
|
|
|
assert(pg->flush_actions.size() == 2);
|
|
|
|
|
for (auto & fa: pg->flush_actions)
|
|
|
|
|
{
|
|
|
|
|
assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 1);
|
|
|
|
|
assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2);
|
|
|
|
|
assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 2u));
|
|
|
|
|
}
|
|
|
|
|
assert(pg->target_history == std::vector<std::vector<osd_num_t>>({ { 1, 3 } }));
|
|
|
|
|
delete pg->peering_state;
|
|
|
|
|
delete pg;
|
|
|
|
|
|
|
|
|
|
@@ -3216,11 +3217,10 @@ void test_peering_partial_unstable_ec_rolls_back()
|
|
|
|
|
delete pg;
|
|
|
|
|
|
|
|
|
|
// ============ D: an unstable chain that was never whole. v2 exists on roles 0 and 1
|
|
|
|
|
// only - pg_data_size pieces, but EC 2+2 needs all 4 to decode, and the object never
|
|
|
|
|
// existed at any stable version: this is what a refused write to a fresh object
|
|
|
|
|
// leaves behind (seed 1996). The old rule (n_roles >= pg_data_size) targeted such a
|
|
|
|
|
// chain and peering-flush committed it; the new rule must refuse v2 and delete the
|
|
|
|
|
// fragments entirely ============
|
|
|
|
|
// only - pg_data_size pieces of a degraded 2-member set, and the object never existed
|
|
|
|
|
// at any stable version: this is what a refused write to a fresh object leaves behind
|
|
|
|
|
// (seed 1996). The pieces are recoverable and may be stable somewhere already, so v2
|
|
|
|
|
// is committed and the object exists ============
|
|
|
|
|
|
|
|
|
|
pg = make_pg();
|
|
|
|
|
pg->pg_cursize = 2;
|
|
|
|
|
@@ -3230,23 +3230,371 @@ void test_peering_partial_unstable_ec_rolls_back()
|
|
|
|
|
add_list(pg, 2, { { 1, 2 } }, 0);
|
|
|
|
|
pg->calc_object_states(0);
|
|
|
|
|
|
|
|
|
|
// Nothing is committed: the object stays absent, both fragments are deleted
|
|
|
|
|
assert(pg->total_count == 0 && pg->clean_count == 0);
|
|
|
|
|
// v2 is committed: the object exists, both fragments are stabilized, their holders are in the history
|
|
|
|
|
assert(pg->total_count == 1 && pg->clean_count == 1);
|
|
|
|
|
assert(pg->state & PG_HAS_UNCLEAN);
|
|
|
|
|
assert(!(pg->state & PG_HAS_INCOMPLETE));
|
|
|
|
|
assert(!pg->ver_override.count(oid));
|
|
|
|
|
assert(pg->flush_actions.size() == 2);
|
|
|
|
|
for (auto & fa: pg->flush_actions)
|
|
|
|
|
{
|
|
|
|
|
assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 0);
|
|
|
|
|
assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2);
|
|
|
|
|
assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 1u));
|
|
|
|
|
}
|
|
|
|
|
assert(pg->target_history == std::vector<std::vector<osd_num_t>>({ { 1, 2 } }));
|
|
|
|
|
delete pg->peering_state;
|
|
|
|
|
delete pg;
|
|
|
|
|
|
|
|
|
|
printf("test_peering_partial_unstable_ec_rolls_back passed\n");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Complete a captured blockstore op like the store does: through a copy of its
|
|
|
|
|
// callback, because some callbacks delete the op that holds them
|
|
|
|
|
static void bs_complete(blockstore_op_t *op, int retval)
|
|
|
|
|
{
|
|
|
|
|
op->retval = retval;
|
|
|
|
|
std::function<void(blockstore_op_t*)>(op->callback)(op);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none, and its role if not osd-1
|
|
|
|
|
struct ec_part_t
|
|
|
|
|
{
|
|
|
|
|
uint64_t stable, unstable;
|
|
|
|
|
int role = -1;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
// EC <data>+<parity> pool with <pg_minsize>, PG 1/1 on <osd_set> or OSDs 1..pg_size, OSD n holds role n-1
|
|
|
|
|
struct ec_scheme_t
|
|
|
|
|
{
|
|
|
|
|
int data, parity, pg_minsize;
|
|
|
|
|
std::vector<osd_num_t> osd_set = {};
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
struct ec_peering_t
|
|
|
|
|
{
|
|
|
|
|
bool active = false, incomplete = false;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static void start_ec_all(osd_test_fixture_t & f, ec_scheme_t ec, osd_num_t primary, const json11::Json & history)
|
|
|
|
|
{
|
|
|
|
|
f.configure_ec_pool(ec.data, ec.parity, primary, ec.pg_minsize);
|
|
|
|
|
if (ec.osd_set.size())
|
|
|
|
|
{
|
|
|
|
|
f.st_cli->set("/vitastor/pg/config", json11::Json::object {
|
|
|
|
|
{ "items", json11::Json::object { { "1", json11::Json::object { { "1", json11::Json::object {
|
|
|
|
|
{ "osd_set", ec.osd_set },
|
|
|
|
|
{ "primary", primary },
|
|
|
|
|
} } } } } },
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
if (!history.is_null())
|
|
|
|
|
f.st_cli->set("/vitastor/pg/history/1/1", history);
|
|
|
|
|
f.start(json11::Json::object {
|
|
|
|
|
{ "osd_num", primary },
|
|
|
|
|
{ "etcd_address", "127.0.0.1:2379" },
|
|
|
|
|
{ "immediate_commit", "all" },
|
|
|
|
|
{ "block_size", 131072 },
|
|
|
|
|
{ "bitmap_granularity", 4096 },
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static json11::Json etcd_pg_history(osd_test_fixture_t & f)
|
|
|
|
|
{
|
|
|
|
|
auto hist_it = f.etcd_store.data.find("/vitastor/pg/history/1/1");
|
|
|
|
|
if (hist_it == f.etcd_store.data.end())
|
|
|
|
|
return json11::Json();
|
|
|
|
|
std::string json_err;
|
|
|
|
|
return json11::Json::parse(hist_it->second.value, json_err);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static bool has_history_set(const json11::Json & history, json11::Json::array set)
|
|
|
|
|
{
|
|
|
|
|
auto & sets = history["osd_sets"].array_items();
|
|
|
|
|
return std::find(sets.begin(), sets.end(), json11::Json(set)) != sets.end();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// One peering of PG 1/1 by <primary> with <peers> up and object A as in <parts>, immediate_commit=all.
|
|
|
|
|
// Applies the stabilize and rollback ops peering sends. With <drop>, the primary loses its peers right
|
|
|
|
|
// after its own op completes, so the peers keep their parts. <history> is the PG history in etcd,
|
|
|
|
|
// carried between peerings
|
|
|
|
|
static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map<osd_num_t, ec_part_t> & parts, osd_num_t primary,
|
|
|
|
|
std::vector<osd_num_t> peers, bool drop, json11::Json & history)
|
|
|
|
|
{
|
|
|
|
|
inode_t ino = INODE_WITH_POOL(1, 1);
|
|
|
|
|
ec_peering_t res;
|
|
|
|
|
auto list = [&](osd_num_t osd)
|
|
|
|
|
{
|
|
|
|
|
std::vector<obj_ver_id> l;
|
|
|
|
|
object_id oid = { .inode = ino, .stripe = (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1) };
|
|
|
|
|
if (parts[osd].stable)
|
|
|
|
|
l.push_back({ oid, parts[osd].stable });
|
|
|
|
|
if (parts[osd].unstable)
|
|
|
|
|
l.push_back({ oid, parts[osd].unstable });
|
|
|
|
|
return l;
|
|
|
|
|
};
|
|
|
|
|
auto apply = [&](osd_num_t osd, bool stabilize, obj_ver_id *ov)
|
|
|
|
|
{
|
|
|
|
|
assert(ov->oid.inode == ino && ov->oid.stripe == (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1));
|
|
|
|
|
if (stabilize)
|
|
|
|
|
{
|
|
|
|
|
assert(ov->version == parts[osd].unstable);
|
|
|
|
|
parts[osd].stable = parts[osd].unstable;
|
|
|
|
|
}
|
|
|
|
|
else
|
|
|
|
|
assert(ov->version == parts[osd].stable);
|
|
|
|
|
parts[osd].unstable = 0;
|
|
|
|
|
};
|
|
|
|
|
osd_test_fixture_t f;
|
|
|
|
|
start_ec_all(f, ec, primary, history);
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.connect_peer(p);
|
|
|
|
|
auto & pg = f.pg(1, 1);
|
|
|
|
|
if (pg.state == PG_PEERING)
|
|
|
|
|
{
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.reply_peer_list(p, list(p), parts[p].stable ? 1 : 0);
|
|
|
|
|
f.reply_local_list(list(primary), parts[primary].stable ? 1 : 0);
|
|
|
|
|
f.pump();
|
|
|
|
|
}
|
|
|
|
|
res.incomplete = pg.state & PG_HAS_INCOMPLETE;
|
|
|
|
|
res.active = pg.state & PG_ACTIVE;
|
|
|
|
|
printf(" OSD %ju peers with %zu more OSD(s): PG 1/1 state=0x%x\n", primary, peers.size(), pg.state);
|
|
|
|
|
bool progress = res.active;
|
|
|
|
|
while (progress)
|
|
|
|
|
{
|
|
|
|
|
progress = false;
|
|
|
|
|
for (int opcode: { BS_OP_STABLE, BS_OP_ROLLBACK })
|
|
|
|
|
{
|
|
|
|
|
if (auto *op = f.bs->take(opcode, false))
|
|
|
|
|
{
|
|
|
|
|
assert(op->len == 1);
|
|
|
|
|
apply(primary, opcode == BS_OP_STABLE, (obj_ver_id*)op->buf);
|
|
|
|
|
bs_complete(op, 0);
|
|
|
|
|
progress = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (drop)
|
|
|
|
|
{
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.fail_peer_connection(p);
|
|
|
|
|
f.pump();
|
|
|
|
|
f.complete_peering_empty();
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
{
|
|
|
|
|
std::vector<osd_op_t*> ops;
|
|
|
|
|
for (auto & kv: f.peer(p)->sent_ops)
|
|
|
|
|
if (kv.second->req.hdr.opcode == OSD_OP_SEC_STABILIZE || kv.second->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)
|
|
|
|
|
ops.push_back(kv.second);
|
|
|
|
|
for (auto *op: ops)
|
|
|
|
|
{
|
|
|
|
|
assert(op->req.sec_stab.len == sizeof(obj_ver_id));
|
|
|
|
|
apply(p, op->req.hdr.opcode == OSD_OP_SEC_STABILIZE, (obj_ver_id*)op->iov.buf[0].iov_base);
|
|
|
|
|
f.peer_complete(op, 0);
|
|
|
|
|
progress = true;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
f.pump();
|
|
|
|
|
}
|
|
|
|
|
history = etcd_pg_history(f);
|
|
|
|
|
for (auto & p: parts)
|
|
|
|
|
printf(" OSD %ju: stable v%jx, unstable v%jx\n", p.first, p.second.stable, p.second.unstable);
|
|
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// After the primary (OSD 1) lost its peers, the others peer without it, then it comes back.
|
|
|
|
|
// Prints and returns whether the PG was active without OSD 1 and whether object A is complete in the end
|
|
|
|
|
static void peer_without_and_with_osd1(ec_scheme_t ec, std::map<osd_num_t, ec_part_t> & parts,
|
|
|
|
|
json11::Json & history, bool & active_without, bool & complete)
|
|
|
|
|
{
|
|
|
|
|
int pg_size = ec.data+ec.parity;
|
|
|
|
|
std::vector<osd_num_t> others;
|
|
|
|
|
for (osd_num_t osd = 3; osd <= pg_size; osd++)
|
|
|
|
|
others.push_back(osd);
|
|
|
|
|
active_without = peer_ec_all(ec, parts, 2, others, false, history).active;
|
|
|
|
|
others.push_back(1);
|
|
|
|
|
auto r = peer_ec_all(ec, parts, 2, others, false, history);
|
|
|
|
|
assert(r.active);
|
|
|
|
|
complete = !r.incomplete;
|
|
|
|
|
printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", complete ? "yes" : "no");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Version 2 of object A is written to OSDs 1..<written> of an EC PG, unstable. OSD 1 peers, flushes it and
|
|
|
|
|
// loses its peers right after its own part is flushed. The others peer without it, and then it comes back.
|
|
|
|
|
// When OSD 1 has stabilized its part, the others must not roll version 2 back: its stable part on OSD 1
|
|
|
|
|
// would then make the object incomplete. While they hold pg_data_size parts of it, the PG must be active
|
|
|
|
|
void test_peering_flush_primary_lost(ec_scheme_t ec, int written)
|
|
|
|
|
{
|
|
|
|
|
printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d]\n", ec.data, ec.parity, ec.pg_minsize, written);
|
|
|
|
|
|
|
|
|
|
std::map<osd_num_t, ec_part_t> parts;
|
|
|
|
|
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
|
|
|
|
|
parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) };
|
|
|
|
|
json11::Json history;
|
|
|
|
|
std::vector<osd_num_t> peers;
|
|
|
|
|
for (osd_num_t osd = 2; osd <= ec.data+ec.parity; osd++)
|
|
|
|
|
peers.push_back(osd);
|
|
|
|
|
peer_ec_all(ec, parts, 1, peers, true, history);
|
|
|
|
|
bool active_without = false, complete = false;
|
|
|
|
|
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
|
|
|
|
|
assert(complete);
|
|
|
|
|
assert(active_without || written == ec.data);
|
|
|
|
|
|
|
|
|
|
printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d] passed\n", ec.data, ec.parity, ec.pg_minsize, written);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Version 2 of object A is written to OSDs 1..<written> of an EC PG, unstable, and a peering flush commits or
|
|
|
|
|
// rolls it back on all OSDs. Then OSD <lost> dies before recovery. The PG must not activate with object A
|
|
|
|
|
// incomplete, and must activate while the other OSDs hold pg_data_size parts of version 2
|
|
|
|
|
void test_peering_flush_then_holder_lost(ec_scheme_t ec, int written, osd_num_t lost)
|
|
|
|
|
{
|
|
|
|
|
printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost]\n",
|
|
|
|
|
ec.data, ec.parity, ec.pg_minsize, written, lost);
|
|
|
|
|
|
|
|
|
|
std::map<osd_num_t, ec_part_t> parts;
|
|
|
|
|
std::vector<osd_num_t> peers, others;
|
|
|
|
|
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
|
|
|
|
|
{
|
|
|
|
|
parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) };
|
|
|
|
|
if (osd != 1)
|
|
|
|
|
peers.push_back(osd);
|
|
|
|
|
if (osd != 1 && osd != lost)
|
|
|
|
|
others.push_back(osd);
|
|
|
|
|
}
|
|
|
|
|
json11::Json history;
|
|
|
|
|
peer_ec_all(ec, parts, 1, peers, false, history);
|
|
|
|
|
auto r = peer_ec_all(ec, parts, 1, others, false, history);
|
|
|
|
|
printf(" PG active without OSD %ju: %s, object incomplete: %s\n", lost, r.active ? "yes" : "no", r.incomplete ? "yes" : "no");
|
|
|
|
|
assert(!r.incomplete);
|
|
|
|
|
assert(r.active || written-1 < ec.data);
|
|
|
|
|
|
|
|
|
|
printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost] passed\n",
|
|
|
|
|
ec.data, ec.parity, ec.pg_minsize, written, lost);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Version 2 of object A is written to OSDs 1..<written> of a degraded EC PG, unstable, over a stable version 1
|
|
|
|
|
// on all OSDs with <older>, or as a new object. OSD 1 peers with them, flushes it and loses its peers right
|
|
|
|
|
// after its own part is flushed. All other OSDs peer without it, and then it comes back. The object must end
|
|
|
|
|
// complete or absent, never incomplete with a lone stable part
|
|
|
|
|
void test_peering_flush_degraded_primary_lost(ec_scheme_t ec, int written, bool older)
|
|
|
|
|
{
|
|
|
|
|
printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s]\n",
|
|
|
|
|
ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object");
|
|
|
|
|
|
|
|
|
|
std::map<osd_num_t, ec_part_t> parts;
|
|
|
|
|
std::vector<osd_num_t> peers;
|
|
|
|
|
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
|
|
|
|
|
{
|
|
|
|
|
parts[osd] = { (uint64_t)(older ? 1 : 0), (uint64_t)(osd <= written ? 2 : 0) };
|
|
|
|
|
if (osd != 1 && osd <= written)
|
|
|
|
|
peers.push_back(osd);
|
|
|
|
|
}
|
|
|
|
|
json11::Json history;
|
|
|
|
|
peer_ec_all(ec, parts, 1, peers, true, history);
|
|
|
|
|
bool active_without = false, complete = false;
|
|
|
|
|
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
|
|
|
|
|
assert(complete);
|
|
|
|
|
assert(active_without || written == ec.data);
|
|
|
|
|
|
|
|
|
|
printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s] passed\n",
|
|
|
|
|
ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// EC 2+1 PG 1/1 has moved from OSDs 1, 2, 3 to 1, 2, 4, and the old set is in the history. Version 2 of
|
|
|
|
|
// object A is unstable on OSD 1 (role 0), on OSD 3 (role 2, outside the current set) and, with <on_osd2>,
|
|
|
|
|
// on OSD 2 (role 1), over a stable version 1 on OSDs 1, 2 and 3. OSD 1 peers, flushes it and loses its peers
|
|
|
|
|
// right after its own part is flushed. The holders of version 2, OSD 3 included, must be in the history.
|
|
|
|
|
// OSDs 2, 3 and 4 peer without OSD 1, and then it comes back: the object must be complete
|
|
|
|
|
void test_peering_flush_misplaced_primary_lost(bool on_osd2)
|
|
|
|
|
{
|
|
|
|
|
printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3]\n", on_osd2 ? "2," : "");
|
|
|
|
|
|
|
|
|
|
ec_scheme_t ec = { 2, 1, 2, { 1, 2, 4 } };
|
|
|
|
|
std::map<osd_num_t, ec_part_t> parts = {
|
|
|
|
|
{ 1, { 1, 2 } },
|
|
|
|
|
{ 2, { 1, (uint64_t)(on_osd2 ? 2 : 0) } },
|
|
|
|
|
{ 3, { 1, 2, 2 } },
|
|
|
|
|
{ 4, { 0, 0, 2 } },
|
|
|
|
|
};
|
|
|
|
|
json11::Json history = json11::Json::object {
|
|
|
|
|
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2, 3 } } },
|
|
|
|
|
};
|
|
|
|
|
peer_ec_all(ec, parts, 1, { 2, 3, 4 }, true, history);
|
|
|
|
|
assert(parts[1].stable == 2);
|
|
|
|
|
assert(has_history_set(history, on_osd2 ? json11::Json::array { 1, 2, 3 } : json11::Json::array { 1, 3 }));
|
|
|
|
|
bool active_without = peer_ec_all(ec, parts, 2, { 3, 4 }, false, history).active;
|
|
|
|
|
auto r = peer_ec_all(ec, parts, 2, { 1, 3, 4 }, false, history);
|
|
|
|
|
printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", r.incomplete ? "no" : "yes");
|
|
|
|
|
assert(r.active && !r.incomplete);
|
|
|
|
|
assert(active_without || !on_osd2);
|
|
|
|
|
|
|
|
|
|
printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3] passed\n", on_osd2 ? "2," : "");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// The same for a client write to a degraded PG of pg_minsize OSDs: OSD 1 writes version 2 of object A to them
|
|
|
|
|
// all, stabilizes its own part and loses its peers before the others stabilize theirs. The degraded OSD set is
|
|
|
|
|
// in the PG history before the write
|
|
|
|
|
void test_degraded_write_primary_lost(ec_scheme_t ec)
|
|
|
|
|
{
|
|
|
|
|
printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d]\n", ec.data, ec.parity, ec.pg_minsize);
|
|
|
|
|
|
|
|
|
|
inode_t ino = INODE_WITH_POOL(1, 1);
|
|
|
|
|
int pg_size = ec.data+ec.parity;
|
|
|
|
|
std::map<osd_num_t, ec_part_t> parts;
|
|
|
|
|
for (osd_num_t osd = 1; osd <= pg_size; osd++)
|
|
|
|
|
parts[osd] = { 1, 0 };
|
|
|
|
|
json11::Json history;
|
|
|
|
|
{
|
|
|
|
|
osd_test_fixture_t f;
|
|
|
|
|
start_ec_all(f, ec, 1, history);
|
|
|
|
|
std::vector<osd_num_t> peers;
|
|
|
|
|
for (osd_num_t osd = 2; osd <= ec.pg_minsize; osd++)
|
|
|
|
|
peers.push_back(osd);
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.connect_peer(p);
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.reply_peer_list(p, { { { ino, p-1 }, 1 } }, 1);
|
|
|
|
|
f.reply_local_list({ { { ino, 0 }, 1 } }, 1);
|
|
|
|
|
f.pump();
|
|
|
|
|
assert((f.pg(1, 1).state & (PG_ACTIVE|PG_DEGRADED)) == (PG_ACTIVE|PG_DEGRADED));
|
|
|
|
|
auto *wr = make_write_op(ino, 0, 131072*ec.data, 0xab);
|
|
|
|
|
int retval = 0;
|
|
|
|
|
wr->callback = [&](osd_op_t *op) { retval = op->reply.hdr.retval; };
|
|
|
|
|
f.exec(wr);
|
|
|
|
|
f.bs_zero_read_ok(1);
|
|
|
|
|
f.pump();
|
|
|
|
|
json11::Json::array cur_set;
|
|
|
|
|
for (osd_num_t osd = 1; osd <= ec.pg_minsize; osd++)
|
|
|
|
|
cur_set.push_back(osd);
|
|
|
|
|
assert(has_history_set(etcd_pg_history(f), cur_set));
|
|
|
|
|
auto *lw = f.bs->take(BS_OP_WRITE);
|
|
|
|
|
uint64_t v2 = lw->version;
|
|
|
|
|
bs_complete(lw, lw->len);
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.peer_write_ok(p, OSD_OP_SEC_WRITE, v2);
|
|
|
|
|
f.pump();
|
|
|
|
|
parts[1].stable = v2;
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
parts[p].unstable = v2;
|
|
|
|
|
bs_complete(f.bs->take(BS_OP_STABLE), 0);
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
assert(f.peer_has_sent_op(p, OSD_OP_SEC_STABILIZE));
|
|
|
|
|
for (auto p: peers)
|
|
|
|
|
f.fail_peer_connection(p);
|
|
|
|
|
f.pump();
|
|
|
|
|
f.complete_peering_empty();
|
|
|
|
|
history = etcd_pg_history(f);
|
|
|
|
|
delete wr;
|
|
|
|
|
}
|
|
|
|
|
bool active_without = false, complete = false;
|
|
|
|
|
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
|
|
|
|
|
assert(complete);
|
|
|
|
|
assert(active_without || ec.pg_minsize == ec.data);
|
|
|
|
|
|
|
|
|
|
printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d] passed\n", ec.data, ec.parity, ec.pg_minsize);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// A dispatched SYNC holds its stabilize snapshot; a delete of a covered object must
|
|
|
|
|
// wait after its fact read while the snapshot is in flight (SEC_STABILIZE of a version
|
|
|
|
|
// the delete removes would answer -ENOENT) and must resume when the SYNC finishes.
|
|
|
|
|
@@ -3760,6 +4108,25 @@ int main(int narg, char *args[])
|
|
|
|
|
test_pg_epoch_bump_above_reported_epoch();
|
|
|
|
|
test_chained_read_large_chain_info();
|
|
|
|
|
test_peering_partial_unstable_ec_rolls_back();
|
|
|
|
|
test_peering_flush_primary_lost({ 2, 1, 2 }, 2);
|
|
|
|
|
test_peering_flush_primary_lost({ 2, 1, 2 }, 3);
|
|
|
|
|
test_peering_flush_primary_lost({ 3, 2, 4 }, 3);
|
|
|
|
|
test_peering_flush_primary_lost({ 3, 2, 4 }, 4);
|
|
|
|
|
test_peering_flush_primary_lost({ 3, 2, 4 }, 5);
|
|
|
|
|
test_degraded_write_primary_lost({ 2, 1, 2 });
|
|
|
|
|
test_degraded_write_primary_lost({ 3, 2, 4 });
|
|
|
|
|
test_peering_flush_then_holder_lost({ 2, 1, 2 }, 2, 2);
|
|
|
|
|
test_peering_flush_then_holder_lost({ 2, 1, 2 }, 3, 2);
|
|
|
|
|
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 4, 3);
|
|
|
|
|
test_peering_flush_then_holder_lost({ 3, 2, 3 }, 3, 2);
|
|
|
|
|
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 3, 2);
|
|
|
|
|
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 5, 3);
|
|
|
|
|
test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, false);
|
|
|
|
|
test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, true);
|
|
|
|
|
test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, false);
|
|
|
|
|
test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, true);
|
|
|
|
|
test_peering_flush_misplaced_primary_lost(true);
|
|
|
|
|
test_peering_flush_misplaced_primary_lost(false);
|
|
|
|
|
test_delete_waits_for_stabilize_snapshot();
|
|
|
|
|
test_delete_waits_only_for_covered_objects();
|
|
|
|
|
test_replicated_add_object_to_set_degraded();
|
|
|
|
|
|