osd: Keep the holders of unstable EC versions in the PG history and accept them from pg_data_size roles #131

Merged
vitalif merged 4 commits from koolkhel/vitastor:peering-flush-minsize into master 2026-11-19 13:58:37 +03:00
8 changed files with 459 additions and 34 deletions

View File

@@ -329,6 +329,7 @@ class osd_t
// flushing, recovery and backfill
void submit_pg_flush_ops(pg_t & pg);
void submit_flush_batch(pg_t & pg);
void handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval);
bool submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t peer_osd, int count, obj_ver_id *data);
bool pick_next_recovery(osd_recovery_op_t &op);

View File

@@ -45,6 +45,23 @@ void osd_t::submit_pg_flush_ops(pg_t & pg)
first = false;
it++;
}
if (pg.epoch > pg.reported_epoch)
{
// Flushes wait until the history with the holders of their versions is reported
if (pg.epoch > pg.reporting_epoch)
{
this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num });
pg.history_changed = true;
report_pg_states();
}
return;
}
submit_flush_batch(pg);
}
void osd_t::submit_flush_batch(pg_t & pg)
{
pg_flush_batch_t *fb = pg.flush_batch;
for (auto & l: fb->rollback_lists)
{
if (l.second.size() > 0)

View File

@@ -354,7 +354,14 @@ bool osd_t::continue_pg_peering(pg_t & pg)
}
if (pg.peering_state->lists_done)
{
auto history_size = pg.target_history.size();
pg.calc_object_states(log_level);
if (pg.target_history.size() != history_size)
{
// Reported with the epoch before the first write or flush. Like there: otherwise
// the PG reconciling loop sees the old history and stops the PG
st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history;
}
report_pg_state(pg);
schedule_scrub(pg);
inconsistent_objects += pg.inconsistent_objects.size();

View File

@@ -110,11 +110,11 @@ void pg_obj_state_check_t::start_object()
void pg_obj_state_check_t::handle_version()
{
// A version becomes target only if it's stable (successfully written to the previous set)
// or unstable but full. We could target any recoverable version, but in this case some
// client's writes written to OSD set A may actually land on a smaller subset of A.
// And then client can't be sure that the pg_minsize guarantee actually holds.
if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_size))
// A version becomes target if it's stable (possibly acked) or unstable but recoverable.
// A higher threshold than the history check (pg_data_size) would let a peering without
// some holders roll back a version already stabilized on them; holders of unstable
// target versions go to the history, so such a peering doesn't activate instead
if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_data_size))
{
target_ver = last_ver;
ver_end = list_pos;
@@ -178,9 +178,9 @@ void pg_obj_state_check_t::handle_version()
void pg_obj_state_check_t::finish_object()
{
if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_size))
if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_data_size))
{
// Stable (possibly acked) or fully materialized unstable - see handle_version()
// Stable (possibly acked) or recoverable unstable - see handle_version()
target_ver = last_ver;
ver_end = list_pos;
}
@@ -200,6 +200,7 @@ void pg_obj_state_check_t::finish_object()
{
pg->state |= PG_HAS_UNCLEAN;
std::unordered_map<obj_piece_id_t, obj_piece_ver_t> pieces;
bool stabilize_target = false;
for (int i = obj_start; i < obj_end; i++)
{
auto & pcs = pieces[(obj_piece_id_t){ .oid = list[i].oid, .osd_num = list[i].osd_num }];
@@ -232,9 +233,27 @@ void pg_obj_state_check_t::finish_object()
{
act.make_stable = true;
act.stable_to = pcs.max_ver > target_ver ? pcs.max_target : pcs.max_ver;
stabilize_target = stabilize_target || act.stable_to == target_ver;
}
}
}
if (stabilize_target)
{
// The flush stabilizes the target version one OSD at a time. Put the OSDs holding
// it into the history, so that a peering without some of them doesn't roll it back
std::vector<osd_num_t> holders;
for (int i = ver_start; i < ver_end; i++)
{
holders.push_back(list[i].osd_num);
}
std::sort(holders.begin(), holders.end());
holders.erase(std::unique(holders.begin(), holders.end()), holders.end());
auto it = std::lower_bound(pg->target_history.begin(), pg->target_history.end(), holders);
if (it == pg->target_history.end() || *it != holders)
{
pg->target_history.insert(it, holders);
}
}
}
if (!target_ver)
{

View File

@@ -79,6 +79,7 @@ struct pg_flush_batch_t
{
std::map<osd_num_t, std::vector<obj_ver_id>> rollback_lists;
std::map<osd_num_t, std::vector<obj_ver_id>> stable_lists;
// 0 while the batch waits for its epoch to be reported
int flush_ops = 0, flush_done = 0, flush_errors = 0;
int flush_objects = 0;
};

View File

@@ -545,6 +545,19 @@ void osd_t::on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num)
}
}
}
pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num });
if (pg_it != pgs.end() && pg_it->second.flush_batch && !pg_it->second.flush_batch->flush_ops)
{
// A flush batch waits for the epoch too, and holds a repeer or a stop like a write
if (pg_it->second.state & PG_ACTIVE)
submit_flush_batch(pg_it->second);
else
{
delete pg_it->second.flush_batch;
pg_it->second.flush_batch = NULL;
continue_pg(pg_it->second);
}
}
}
}

View File

@@ -3107,16 +3107,14 @@ void test_pg_epoch_bump_above_reported_epoch()
printf("test_pg_epoch_bump_above_reported_epoch passed\n");
}
// A partial unstable chain left by a failed/rejected EC write must not become the
// committed version even when it holds pg_data_size pieces: that would turn a
// possibly-nonexistent write into committed data with zero redundancy. Peering must
// fall back to the older stable version and roll the fragments back. A stable
// fragment, on the other hand, is possibly-acked - the OSD can never know otherwise
// - so it stays the target and is never rolled back, incomplete or not.
// A partial chain of pg_data_size pieces is not committed either, even when a
// degraded set holds a fragment per member: an EC 2+2 pair on the two data roles
// of a 2-member set still cannot decode (subcase D: refused-write fragments of a
// fresh object - peering must delete them, not commit).
// A partial unstable chain left by a failed/rejected EC write becomes the committed
// version when it holds pg_data_size pieces: a flush may have stabilized some of them
// already, and the history check lets a later peering activate with pg_data_size OSDs
// of a set, so a higher threshold would roll back a version stable on missing OSDs.
// The holders of the chain go to the PG history instead, so a peering without
// pg_data_size of them doesn't activate. A stable fragment is possibly-acked - the OSD
// can never know otherwise - so it stays the target and is never rolled back,
// incomplete or not.
//
// Layout: EC 2+2, PG 1/1, cur_set = role 0..3 => OSD 1..4. Object INODE_WITH_POOL(1,1)
// is stable at v1 on all roles; a rejected write left v2 unstable on role 0 (OSD 1)
@@ -3157,7 +3155,7 @@ void test_peering_partial_unstable_ec_rolls_back()
};
};
// ============ A: partial unstable v2 falls back to stable v1 ============
// ============ A: partial unstable v2 on pg_data_size roles is committed ============
pg_t *pg = make_pg();
add_list(pg, 1, { { 0, 1 }, { 0, 2 } }, 1);
@@ -3166,18 +3164,21 @@ void test_peering_partial_unstable_ec_rolls_back()
add_list(pg, 4, { { 3, 1 } }, 1);
pg->calc_object_states(0);
// v1 is the target: the object is clean at v1, reads see v1, fragments roll back
assert(pg->total_count == 1 && pg->clean_count == 1);
assert(!(pg->state & (PG_HAS_INCOMPLETE|PG_HAS_DEGRADED)));
// v2 is the target: the object is degraded at v2, the fragments are stabilized, and their
// holders are in the history, so losing one before the PG is clean keeps the PG inactive
assert(pg->total_count == 1 && pg->clean_count == 0);
assert(pg->state & PG_HAS_DEGRADED);
assert(!(pg->state & PG_HAS_INCOMPLETE));
assert(pg->state & PG_HAS_UNCLEAN);
assert(!find_object_state(*pg, oid));
assert(pg->ver_override.count(oid) && pg->ver_override[oid] == 1);
assert(find_object_state(*pg, oid));
assert(!pg->ver_override.count(oid));
assert(pg->flush_actions.size() == 2);
for (auto & fa: pg->flush_actions)
{
assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 1);
assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2);
assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 2u));
}
assert(pg->target_history == std::vector<std::vector<osd_num_t>>({ { 1, 3 } }));
delete pg->peering_state;
delete pg;
@@ -3216,11 +3217,10 @@ void test_peering_partial_unstable_ec_rolls_back()
delete pg;
// ============ D: an unstable chain that was never whole. v2 exists on roles 0 and 1
// only - pg_data_size pieces, but EC 2+2 needs all 4 to decode, and the object never
// existed at any stable version: this is what a refused write to a fresh object
// leaves behind (seed 1996). The old rule (n_roles >= pg_data_size) targeted such a
// chain and peering-flush committed it; the new rule must refuse v2 and delete the
// fragments entirely ============
// only - pg_data_size pieces of a degraded 2-member set, and the object never existed
// at any stable version: this is what a refused write to a fresh object leaves behind
// (seed 1996). The pieces are recoverable and may be stable somewhere already, so v2
// is committed and the object exists ============
pg = make_pg();
pg->pg_cursize = 2;
@@ -3230,23 +3230,371 @@ void test_peering_partial_unstable_ec_rolls_back()
add_list(pg, 2, { { 1, 2 } }, 0);
pg->calc_object_states(0);
// Nothing is committed: the object stays absent, both fragments are deleted
assert(pg->total_count == 0 && pg->clean_count == 0);
// v2 is committed: the object exists, both fragments are stabilized, their holders are in the history
assert(pg->total_count == 1 && pg->clean_count == 1);
assert(pg->state & PG_HAS_UNCLEAN);
assert(!(pg->state & PG_HAS_INCOMPLETE));
assert(!pg->ver_override.count(oid));
assert(pg->flush_actions.size() == 2);
for (auto & fa: pg->flush_actions)
{
assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 0);
assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2);
assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 1u));
}
assert(pg->target_history == std::vector<std::vector<osd_num_t>>({ { 1, 2 } }));
delete pg->peering_state;
delete pg;
printf("test_peering_partial_unstable_ec_rolls_back passed\n");
}
// Complete a captured blockstore op like the store does: through a copy of its
// callback, because some callbacks delete the op that holds them
static void bs_complete(blockstore_op_t *op, int retval)
{
op->retval = retval;
std::function<void(blockstore_op_t*)>(op->callback)(op);
}
// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none, and its role if not osd-1
struct ec_part_t
{
uint64_t stable, unstable;
int role = -1;
};
// EC <data>+<parity> pool with <pg_minsize>, PG 1/1 on <osd_set> or OSDs 1..pg_size, OSD n holds role n-1
struct ec_scheme_t
{
int data, parity, pg_minsize;
std::vector<osd_num_t> osd_set = {};
};
struct ec_peering_t
{
bool active = false, incomplete = false;
};
static void start_ec_all(osd_test_fixture_t & f, ec_scheme_t ec, osd_num_t primary, const json11::Json & history)
{
f.configure_ec_pool(ec.data, ec.parity, primary, ec.pg_minsize);
if (ec.osd_set.size())
{
f.st_cli->set("/vitastor/pg/config", json11::Json::object {
{ "items", json11::Json::object { { "1", json11::Json::object { { "1", json11::Json::object {
{ "osd_set", ec.osd_set },
{ "primary", primary },
} } } } } },
});
}
if (!history.is_null())
f.st_cli->set("/vitastor/pg/history/1/1", history);
f.start(json11::Json::object {
{ "osd_num", primary },
{ "etcd_address", "127.0.0.1:2379" },
{ "immediate_commit", "all" },
{ "block_size", 131072 },
{ "bitmap_granularity", 4096 },
});
}
static json11::Json etcd_pg_history(osd_test_fixture_t & f)
{
auto hist_it = f.etcd_store.data.find("/vitastor/pg/history/1/1");
if (hist_it == f.etcd_store.data.end())
return json11::Json();
std::string json_err;
return json11::Json::parse(hist_it->second.value, json_err);
}
static bool has_history_set(const json11::Json & history, json11::Json::array set)
{
auto & sets = history["osd_sets"].array_items();
return std::find(sets.begin(), sets.end(), json11::Json(set)) != sets.end();
}
// One peering of PG 1/1 by <primary> with <peers> up and object A as in <parts>, immediate_commit=all.
// Applies the stabilize and rollback ops peering sends. With <drop>, the primary loses its peers right
// after its own op completes, so the peers keep their parts. <history> is the PG history in etcd,
// carried between peerings
static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map<osd_num_t, ec_part_t> & parts, osd_num_t primary,
std::vector<osd_num_t> peers, bool drop, json11::Json & history)
{
inode_t ino = INODE_WITH_POOL(1, 1);
ec_peering_t res;
auto list = [&](osd_num_t osd)
{
std::vector<obj_ver_id> l;
object_id oid = { .inode = ino, .stripe = (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1) };
if (parts[osd].stable)
l.push_back({ oid, parts[osd].stable });
if (parts[osd].unstable)
l.push_back({ oid, parts[osd].unstable });
return l;
};
auto apply = [&](osd_num_t osd, bool stabilize, obj_ver_id *ov)
{
assert(ov->oid.inode == ino && ov->oid.stripe == (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1));
if (stabilize)
{
assert(ov->version == parts[osd].unstable);
parts[osd].stable = parts[osd].unstable;
}
else
assert(ov->version == parts[osd].stable);
parts[osd].unstable = 0;
};
osd_test_fixture_t f;
start_ec_all(f, ec, primary, history);
for (auto p: peers)
f.connect_peer(p);
auto & pg = f.pg(1, 1);
if (pg.state == PG_PEERING)
{
for (auto p: peers)
f.reply_peer_list(p, list(p), parts[p].stable ? 1 : 0);
f.reply_local_list(list(primary), parts[primary].stable ? 1 : 0);
f.pump();
}
res.incomplete = pg.state & PG_HAS_INCOMPLETE;
res.active = pg.state & PG_ACTIVE;
printf(" OSD %ju peers with %zu more OSD(s): PG 1/1 state=0x%x\n", primary, peers.size(), pg.state);
bool progress = res.active;
while (progress)
{
progress = false;
for (int opcode: { BS_OP_STABLE, BS_OP_ROLLBACK })
{
if (auto *op = f.bs->take(opcode, false))
{
assert(op->len == 1);
apply(primary, opcode == BS_OP_STABLE, (obj_ver_id*)op->buf);
bs_complete(op, 0);
progress = true;
}
}
if (drop)
{
for (auto p: peers)
f.fail_peer_connection(p);
f.pump();
f.complete_peering_empty();
break;
}
for (auto p: peers)
{
std::vector<osd_op_t*> ops;
for (auto & kv: f.peer(p)->sent_ops)
if (kv.second->req.hdr.opcode == OSD_OP_SEC_STABILIZE || kv.second->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)
ops.push_back(kv.second);
for (auto *op: ops)
{
assert(op->req.sec_stab.len == sizeof(obj_ver_id));
apply(p, op->req.hdr.opcode == OSD_OP_SEC_STABILIZE, (obj_ver_id*)op->iov.buf[0].iov_base);
f.peer_complete(op, 0);
progress = true;
}
}
f.pump();
}
history = etcd_pg_history(f);
for (auto & p: parts)
printf(" OSD %ju: stable v%jx, unstable v%jx\n", p.first, p.second.stable, p.second.unstable);
return res;
}
// After the primary (OSD 1) lost its peers, the others peer without it, then it comes back.
// Prints and returns whether the PG was active without OSD 1 and whether object A is complete in the end
static void peer_without_and_with_osd1(ec_scheme_t ec, std::map<osd_num_t, ec_part_t> & parts,
json11::Json & history, bool & active_without, bool & complete)
{
int pg_size = ec.data+ec.parity;
std::vector<osd_num_t> others;
for (osd_num_t osd = 3; osd <= pg_size; osd++)
others.push_back(osd);
active_without = peer_ec_all(ec, parts, 2, others, false, history).active;
others.push_back(1);
auto r = peer_ec_all(ec, parts, 2, others, false, history);
assert(r.active);
complete = !r.incomplete;
printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", complete ? "yes" : "no");
}
// Version 2 of object A is written to OSDs 1..<written> of an EC PG, unstable. OSD 1 peers, flushes it and
// loses its peers right after its own part is flushed. The others peer without it, and then it comes back.
// When OSD 1 has stabilized its part, the others must not roll version 2 back: its stable part on OSD 1
// would then make the object incomplete. While they hold pg_data_size parts of it, the PG must be active
void test_peering_flush_primary_lost(ec_scheme_t ec, int written)
{
printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d]\n", ec.data, ec.parity, ec.pg_minsize, written);
std::map<osd_num_t, ec_part_t> parts;
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) };
json11::Json history;
std::vector<osd_num_t> peers;
for (osd_num_t osd = 2; osd <= ec.data+ec.parity; osd++)
peers.push_back(osd);
peer_ec_all(ec, parts, 1, peers, true, history);
bool active_without = false, complete = false;
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
assert(complete);
assert(active_without || written == ec.data);
printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d] passed\n", ec.data, ec.parity, ec.pg_minsize, written);
}
// Version 2 of object A is written to OSDs 1..<written> of an EC PG, unstable, and a peering flush commits or
// rolls it back on all OSDs. Then OSD <lost> dies before recovery. The PG must not activate with object A
// incomplete, and must activate while the other OSDs hold pg_data_size parts of version 2
void test_peering_flush_then_holder_lost(ec_scheme_t ec, int written, osd_num_t lost)
{
printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost]\n",
ec.data, ec.parity, ec.pg_minsize, written, lost);
std::map<osd_num_t, ec_part_t> parts;
std::vector<osd_num_t> peers, others;
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
{
parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) };
if (osd != 1)
peers.push_back(osd);
if (osd != 1 && osd != lost)
others.push_back(osd);
}
json11::Json history;
peer_ec_all(ec, parts, 1, peers, false, history);
auto r = peer_ec_all(ec, parts, 1, others, false, history);
printf(" PG active without OSD %ju: %s, object incomplete: %s\n", lost, r.active ? "yes" : "no", r.incomplete ? "yes" : "no");
assert(!r.incomplete);
assert(r.active || written-1 < ec.data);
printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost] passed\n",
ec.data, ec.parity, ec.pg_minsize, written, lost);
}
// Version 2 of object A is written to OSDs 1..<written> of a degraded EC PG, unstable, over a stable version 1
// on all OSDs with <older>, or as a new object. OSD 1 peers with them, flushes it and loses its peers right
// after its own part is flushed. All other OSDs peer without it, and then it comes back. The object must end
// complete or absent, never incomplete with a lone stable part
void test_peering_flush_degraded_primary_lost(ec_scheme_t ec, int written, bool older)
{
printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s]\n",
ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object");
std::map<osd_num_t, ec_part_t> parts;
std::vector<osd_num_t> peers;
for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++)
{
parts[osd] = { (uint64_t)(older ? 1 : 0), (uint64_t)(osd <= written ? 2 : 0) };
if (osd != 1 && osd <= written)
peers.push_back(osd);
}
json11::Json history;
peer_ec_all(ec, parts, 1, peers, true, history);
bool active_without = false, complete = false;
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
assert(complete);
assert(active_without || written == ec.data);
printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s] passed\n",
ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object");
}
// EC 2+1 PG 1/1 has moved from OSDs 1, 2, 3 to 1, 2, 4, and the old set is in the history. Version 2 of
// object A is unstable on OSD 1 (role 0), on OSD 3 (role 2, outside the current set) and, with <on_osd2>,
// on OSD 2 (role 1), over a stable version 1 on OSDs 1, 2 and 3. OSD 1 peers, flushes it and loses its peers
// right after its own part is flushed. The holders of version 2, OSD 3 included, must be in the history.
// OSDs 2, 3 and 4 peer without OSD 1, and then it comes back: the object must be complete
void test_peering_flush_misplaced_primary_lost(bool on_osd2)
{
printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3]\n", on_osd2 ? "2," : "");
ec_scheme_t ec = { 2, 1, 2, { 1, 2, 4 } };
std::map<osd_num_t, ec_part_t> parts = {
{ 1, { 1, 2 } },
{ 2, { 1, (uint64_t)(on_osd2 ? 2 : 0) } },
{ 3, { 1, 2, 2 } },
{ 4, { 0, 0, 2 } },
};
json11::Json history = json11::Json::object {
{ "osd_sets", json11::Json::array { json11::Json::array { 1, 2, 3 } } },
};
peer_ec_all(ec, parts, 1, { 2, 3, 4 }, true, history);
assert(parts[1].stable == 2);
assert(has_history_set(history, on_osd2 ? json11::Json::array { 1, 2, 3 } : json11::Json::array { 1, 3 }));
bool active_without = peer_ec_all(ec, parts, 2, { 3, 4 }, false, history).active;
auto r = peer_ec_all(ec, parts, 2, { 1, 3, 4 }, false, history);
printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", r.incomplete ? "no" : "yes");
assert(r.active && !r.incomplete);
assert(active_without || !on_osd2);
printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3] passed\n", on_osd2 ? "2," : "");
}
// The same for a client write to a degraded PG of pg_minsize OSDs: OSD 1 writes version 2 of object A to them
// all, stabilizes its own part and loses its peers before the others stabilize theirs. The degraded OSD set is
// in the PG history before the write
void test_degraded_write_primary_lost(ec_scheme_t ec)
{
printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d]\n", ec.data, ec.parity, ec.pg_minsize);
inode_t ino = INODE_WITH_POOL(1, 1);
int pg_size = ec.data+ec.parity;
std::map<osd_num_t, ec_part_t> parts;
for (osd_num_t osd = 1; osd <= pg_size; osd++)
parts[osd] = { 1, 0 };
json11::Json history;
{
osd_test_fixture_t f;
start_ec_all(f, ec, 1, history);
std::vector<osd_num_t> peers;
for (osd_num_t osd = 2; osd <= ec.pg_minsize; osd++)
peers.push_back(osd);
for (auto p: peers)
f.connect_peer(p);
for (auto p: peers)
f.reply_peer_list(p, { { { ino, p-1 }, 1 } }, 1);
f.reply_local_list({ { { ino, 0 }, 1 } }, 1);
f.pump();
assert((f.pg(1, 1).state & (PG_ACTIVE|PG_DEGRADED)) == (PG_ACTIVE|PG_DEGRADED));
auto *wr = make_write_op(ino, 0, 131072*ec.data, 0xab);
int retval = 0;
wr->callback = [&](osd_op_t *op) { retval = op->reply.hdr.retval; };
f.exec(wr);
f.bs_zero_read_ok(1);
f.pump();
json11::Json::array cur_set;
for (osd_num_t osd = 1; osd <= ec.pg_minsize; osd++)
cur_set.push_back(osd);
assert(has_history_set(etcd_pg_history(f), cur_set));
auto *lw = f.bs->take(BS_OP_WRITE);
uint64_t v2 = lw->version;
bs_complete(lw, lw->len);
for (auto p: peers)
f.peer_write_ok(p, OSD_OP_SEC_WRITE, v2);
f.pump();
parts[1].stable = v2;
for (auto p: peers)
parts[p].unstable = v2;
bs_complete(f.bs->take(BS_OP_STABLE), 0);
for (auto p: peers)
assert(f.peer_has_sent_op(p, OSD_OP_SEC_STABILIZE));
for (auto p: peers)
f.fail_peer_connection(p);
f.pump();
f.complete_peering_empty();
history = etcd_pg_history(f);
delete wr;
}
bool active_without = false, complete = false;
peer_without_and_with_osd1(ec, parts, history, active_without, complete);
assert(complete);
assert(active_without || ec.pg_minsize == ec.data);
printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d] passed\n", ec.data, ec.parity, ec.pg_minsize);
}
// A dispatched SYNC holds its stabilize snapshot; a delete of a covered object must
// wait after its fact read while the snapshot is in flight (SEC_STABILIZE of a version
// the delete removes would answer -ENOENT) and must resume when the SYNC finishes.
@@ -3760,6 +4108,25 @@ int main(int narg, char *args[])
test_pg_epoch_bump_above_reported_epoch();
test_chained_read_large_chain_info();
test_peering_partial_unstable_ec_rolls_back();
test_peering_flush_primary_lost({ 2, 1, 2 }, 2);
test_peering_flush_primary_lost({ 2, 1, 2 }, 3);
test_peering_flush_primary_lost({ 3, 2, 4 }, 3);
test_peering_flush_primary_lost({ 3, 2, 4 }, 4);
test_peering_flush_primary_lost({ 3, 2, 4 }, 5);
test_degraded_write_primary_lost({ 2, 1, 2 });
test_degraded_write_primary_lost({ 3, 2, 4 });
test_peering_flush_then_holder_lost({ 2, 1, 2 }, 2, 2);
test_peering_flush_then_holder_lost({ 2, 1, 2 }, 3, 2);
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 4, 3);
test_peering_flush_then_holder_lost({ 3, 2, 3 }, 3, 2);
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 3, 2);
test_peering_flush_then_holder_lost({ 3, 2, 4 }, 5, 3);
test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, false);
test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, true);
test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, false);
test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, true);
test_peering_flush_misplaced_primary_lost(true);
test_peering_flush_misplaced_primary_lost(false);
test_delete_waits_for_stabilize_snapshot();
test_delete_waits_only_for_covered_objects();
test_replicated_add_object_to_set_degraded();

View File

@@ -132,7 +132,7 @@ struct osd_test_fixture_t
});
}
void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2)
void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2, int pg_minsize = 0)
{
auto pool_id_s = std::to_string(1);
st_cli->set("/vitastor/config/pools", json11::Json::object {
@@ -140,7 +140,7 @@ struct osd_test_fixture_t
{ "name", "pool_1" },
{ "scheme", "ec" },
{ "pg_size", data_chunks+parity_chunks },
{ "pg_minsize", data_chunks },
{ "pg_minsize", pg_minsize ? pg_minsize : data_chunks },
{ "parity_chunks", parity_chunks },
{ "pg_count", 1 },
{ "failure_domain", "osd" },