diff --git a/src/osd/osd.h b/src/osd/osd.h index 0e2579e0..b146565c 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -329,6 +329,7 @@ class osd_t // flushing, recovery and backfill void submit_pg_flush_ops(pg_t & pg); + void submit_flush_batch(pg_t & pg); void handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval); bool submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t peer_osd, int count, obj_ver_id *data); bool pick_next_recovery(osd_recovery_op_t &op); diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 96738cd1..402ba0a4 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -45,6 +45,23 @@ void osd_t::submit_pg_flush_ops(pg_t & pg) first = false; it++; } + if (pg.epoch > pg.reported_epoch) + { + // Flushes wait until the history with the holders of their versions is reported + if (pg.epoch > pg.reporting_epoch) + { + this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num }); + pg.history_changed = true; + report_pg_states(); + } + return; + } + submit_flush_batch(pg); +} + +void osd_t::submit_flush_batch(pg_t & pg) +{ + pg_flush_batch_t *fb = pg.flush_batch; for (auto & l: fb->rollback_lists) { if (l.second.size() > 0) diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index cdffbcf1..1a88da05 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -354,7 +354,14 @@ bool osd_t::continue_pg_peering(pg_t & pg) } if (pg.peering_state->lists_done) { + auto history_size = pg.target_history.size(); pg.calc_object_states(log_level); + if (pg.target_history.size() != history_size) + { + // Reported with the epoch before the first write or flush. Like there: otherwise + // the PG reconciling loop sees the old history and stops the PG + st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history; + } report_pg_state(pg); schedule_scrub(pg); inconsistent_objects += pg.inconsistent_objects.size(); diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index e978041a..0cab0751 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -110,11 +110,11 @@ void pg_obj_state_check_t::start_object() void pg_obj_state_check_t::handle_version() { - // A version becomes target only if it's stable (successfully written to the previous set) - // or unstable but full. We could target any recoverable version, but in this case some - // client's writes written to OSD set A may actually land on a smaller subset of A. - // And then client can't be sure that the pg_minsize guarantee actually holds. - if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_size)) + // A version becomes target if it's stable (possibly acked) or unstable but recoverable. + // A higher threshold than the history check (pg_data_size) would let a peering without + // some holders roll back a version already stabilized on them; holders of unstable + // target versions go to the history, so such a peering doesn't activate instead + if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_data_size)) { target_ver = last_ver; ver_end = list_pos; @@ -178,9 +178,9 @@ void pg_obj_state_check_t::handle_version() void pg_obj_state_check_t::finish_object() { - if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_size)) + if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_data_size)) { - // Stable (possibly acked) or fully materialized unstable - see handle_version() + // Stable (possibly acked) or recoverable unstable - see handle_version() target_ver = last_ver; ver_end = list_pos; } @@ -200,6 +200,7 @@ void pg_obj_state_check_t::finish_object() { pg->state |= PG_HAS_UNCLEAN; std::unordered_map pieces; + bool stabilize_target = false; for (int i = obj_start; i < obj_end; i++) { auto & pcs = pieces[(obj_piece_id_t){ .oid = list[i].oid, .osd_num = list[i].osd_num }]; @@ -232,9 +233,27 @@ void pg_obj_state_check_t::finish_object() { act.make_stable = true; act.stable_to = pcs.max_ver > target_ver ? pcs.max_target : pcs.max_ver; + stabilize_target = stabilize_target || act.stable_to == target_ver; } } } + if (stabilize_target) + { + // The flush stabilizes the target version one OSD at a time. Put the OSDs holding + // it into the history, so that a peering without some of them doesn't roll it back + std::vector holders; + for (int i = ver_start; i < ver_end; i++) + { + holders.push_back(list[i].osd_num); + } + std::sort(holders.begin(), holders.end()); + holders.erase(std::unique(holders.begin(), holders.end()), holders.end()); + auto it = std::lower_bound(pg->target_history.begin(), pg->target_history.end(), holders); + if (it == pg->target_history.end() || *it != holders) + { + pg->target_history.insert(it, holders); + } + } } if (!target_ver) { diff --git a/src/osd/osd_peering_pg.h b/src/osd/osd_peering_pg.h index 3bcc2fb6..cdbd91ef 100644 --- a/src/osd/osd_peering_pg.h +++ b/src/osd/osd_peering_pg.h @@ -79,6 +79,7 @@ struct pg_flush_batch_t { std::map> rollback_lists; std::map> stable_lists; + // 0 while the batch waits for its epoch to be reported int flush_ops = 0, flush_done = 0, flush_errors = 0; int flush_objects = 0; }; diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index 0a70bf01..fa5dd390 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -545,6 +545,19 @@ void osd_t::on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num) } } } + pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num }); + if (pg_it != pgs.end() && pg_it->second.flush_batch && !pg_it->second.flush_batch->flush_ops) + { + // A flush batch waits for the epoch too, and holds a repeer or a stop like a write + if (pg_it->second.state & PG_ACTIVE) + submit_flush_batch(pg_it->second); + else + { + delete pg_it->second.flush_batch; + pg_it->second.flush_batch = NULL; + continue_pg(pg_it->second); + } + } } } diff --git a/src/test/osd_test.cpp b/src/test/osd_test.cpp index 63633d08..1df6133a 100644 --- a/src/test/osd_test.cpp +++ b/src/test/osd_test.cpp @@ -3107,16 +3107,14 @@ void test_pg_epoch_bump_above_reported_epoch() printf("test_pg_epoch_bump_above_reported_epoch passed\n"); } -// A partial unstable chain left by a failed/rejected EC write must not become the -// committed version even when it holds pg_data_size pieces: that would turn a -// possibly-nonexistent write into committed data with zero redundancy. Peering must -// fall back to the older stable version and roll the fragments back. A stable -// fragment, on the other hand, is possibly-acked - the OSD can never know otherwise -// - so it stays the target and is never rolled back, incomplete or not. -// A partial chain of pg_data_size pieces is not committed either, even when a -// degraded set holds a fragment per member: an EC 2+2 pair on the two data roles -// of a 2-member set still cannot decode (subcase D: refused-write fragments of a -// fresh object - peering must delete them, not commit). +// A partial unstable chain left by a failed/rejected EC write becomes the committed +// version when it holds pg_data_size pieces: a flush may have stabilized some of them +// already, and the history check lets a later peering activate with pg_data_size OSDs +// of a set, so a higher threshold would roll back a version stable on missing OSDs. +// The holders of the chain go to the PG history instead, so a peering without +// pg_data_size of them doesn't activate. A stable fragment is possibly-acked - the OSD +// can never know otherwise - so it stays the target and is never rolled back, +// incomplete or not. // // Layout: EC 2+2, PG 1/1, cur_set = role 0..3 => OSD 1..4. Object INODE_WITH_POOL(1,1) // is stable at v1 on all roles; a rejected write left v2 unstable on role 0 (OSD 1) @@ -3157,7 +3155,7 @@ void test_peering_partial_unstable_ec_rolls_back() }; }; - // ============ A: partial unstable v2 falls back to stable v1 ============ + // ============ A: partial unstable v2 on pg_data_size roles is committed ============ pg_t *pg = make_pg(); add_list(pg, 1, { { 0, 1 }, { 0, 2 } }, 1); @@ -3166,18 +3164,21 @@ void test_peering_partial_unstable_ec_rolls_back() add_list(pg, 4, { { 3, 1 } }, 1); pg->calc_object_states(0); - // v1 is the target: the object is clean at v1, reads see v1, fragments roll back - assert(pg->total_count == 1 && pg->clean_count == 1); - assert(!(pg->state & (PG_HAS_INCOMPLETE|PG_HAS_DEGRADED))); + // v2 is the target: the object is degraded at v2, the fragments are stabilized, and their + // holders are in the history, so losing one before the PG is clean keeps the PG inactive + assert(pg->total_count == 1 && pg->clean_count == 0); + assert(pg->state & PG_HAS_DEGRADED); + assert(!(pg->state & PG_HAS_INCOMPLETE)); assert(pg->state & PG_HAS_UNCLEAN); - assert(!find_object_state(*pg, oid)); - assert(pg->ver_override.count(oid) && pg->ver_override[oid] == 1); + assert(find_object_state(*pg, oid)); + assert(!pg->ver_override.count(oid)); assert(pg->flush_actions.size() == 2); for (auto & fa: pg->flush_actions) { - assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 1); + assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2); assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 2u)); } + assert(pg->target_history == std::vector>({ { 1, 3 } })); delete pg->peering_state; delete pg; @@ -3216,11 +3217,10 @@ void test_peering_partial_unstable_ec_rolls_back() delete pg; // ============ D: an unstable chain that was never whole. v2 exists on roles 0 and 1 - // only - pg_data_size pieces, but EC 2+2 needs all 4 to decode, and the object never - // existed at any stable version: this is what a refused write to a fresh object - // leaves behind (seed 1996). The old rule (n_roles >= pg_data_size) targeted such a - // chain and peering-flush committed it; the new rule must refuse v2 and delete the - // fragments entirely ============ + // only - pg_data_size pieces of a degraded 2-member set, and the object never existed + // at any stable version: this is what a refused write to a fresh object leaves behind + // (seed 1996). The pieces are recoverable and may be stable somewhere already, so v2 + // is committed and the object exists ============ pg = make_pg(); pg->pg_cursize = 2; @@ -3230,23 +3230,371 @@ void test_peering_partial_unstable_ec_rolls_back() add_list(pg, 2, { { 1, 2 } }, 0); pg->calc_object_states(0); - // Nothing is committed: the object stays absent, both fragments are deleted - assert(pg->total_count == 0 && pg->clean_count == 0); + // v2 is committed: the object exists, both fragments are stabilized, their holders are in the history + assert(pg->total_count == 1 && pg->clean_count == 1); assert(pg->state & PG_HAS_UNCLEAN); assert(!(pg->state & PG_HAS_INCOMPLETE)); assert(!pg->ver_override.count(oid)); assert(pg->flush_actions.size() == 2); for (auto & fa: pg->flush_actions) { - assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 0); + assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2); assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 1u)); } + assert(pg->target_history == std::vector>({ { 1, 2 } })); delete pg->peering_state; delete pg; printf("test_peering_partial_unstable_ec_rolls_back passed\n"); } +// Complete a captured blockstore op like the store does: through a copy of its +// callback, because some callbacks delete the op that holds them +static void bs_complete(blockstore_op_t *op, int retval) +{ + op->retval = retval; + std::function(op->callback)(op); +} + +// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none, and its role if not osd-1 +struct ec_part_t +{ + uint64_t stable, unstable; + int role = -1; +}; + +// EC + pool with , PG 1/1 on or OSDs 1..pg_size, OSD n holds role n-1 +struct ec_scheme_t +{ + int data, parity, pg_minsize; + std::vector osd_set = {}; +}; + +struct ec_peering_t +{ + bool active = false, incomplete = false; +}; + +static void start_ec_all(osd_test_fixture_t & f, ec_scheme_t ec, osd_num_t primary, const json11::Json & history) +{ + f.configure_ec_pool(ec.data, ec.parity, primary, ec.pg_minsize); + if (ec.osd_set.size()) + { + f.st_cli->set("/vitastor/pg/config", json11::Json::object { + { "items", json11::Json::object { { "1", json11::Json::object { { "1", json11::Json::object { + { "osd_set", ec.osd_set }, + { "primary", primary }, + } } } } } }, + }); + } + if (!history.is_null()) + f.st_cli->set("/vitastor/pg/history/1/1", history); + f.start(json11::Json::object { + { "osd_num", primary }, + { "etcd_address", "127.0.0.1:2379" }, + { "immediate_commit", "all" }, + { "block_size", 131072 }, + { "bitmap_granularity", 4096 }, + }); +} + +static json11::Json etcd_pg_history(osd_test_fixture_t & f) +{ + auto hist_it = f.etcd_store.data.find("/vitastor/pg/history/1/1"); + if (hist_it == f.etcd_store.data.end()) + return json11::Json(); + std::string json_err; + return json11::Json::parse(hist_it->second.value, json_err); +} + +static bool has_history_set(const json11::Json & history, json11::Json::array set) +{ + auto & sets = history["osd_sets"].array_items(); + return std::find(sets.begin(), sets.end(), json11::Json(set)) != sets.end(); +} + +// One peering of PG 1/1 by with up and object A as in , immediate_commit=all. +// Applies the stabilize and rollback ops peering sends. With , the primary loses its peers right +// after its own op completes, so the peers keep their parts. is the PG history in etcd, +// carried between peerings +static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map & parts, osd_num_t primary, + std::vector peers, bool drop, json11::Json & history) +{ + inode_t ino = INODE_WITH_POOL(1, 1); + ec_peering_t res; + auto list = [&](osd_num_t osd) + { + std::vector l; + object_id oid = { .inode = ino, .stripe = (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1) }; + if (parts[osd].stable) + l.push_back({ oid, parts[osd].stable }); + if (parts[osd].unstable) + l.push_back({ oid, parts[osd].unstable }); + return l; + }; + auto apply = [&](osd_num_t osd, bool stabilize, obj_ver_id *ov) + { + assert(ov->oid.inode == ino && ov->oid.stripe == (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1)); + if (stabilize) + { + assert(ov->version == parts[osd].unstable); + parts[osd].stable = parts[osd].unstable; + } + else + assert(ov->version == parts[osd].stable); + parts[osd].unstable = 0; + }; + osd_test_fixture_t f; + start_ec_all(f, ec, primary, history); + for (auto p: peers) + f.connect_peer(p); + auto & pg = f.pg(1, 1); + if (pg.state == PG_PEERING) + { + for (auto p: peers) + f.reply_peer_list(p, list(p), parts[p].stable ? 1 : 0); + f.reply_local_list(list(primary), parts[primary].stable ? 1 : 0); + f.pump(); + } + res.incomplete = pg.state & PG_HAS_INCOMPLETE; + res.active = pg.state & PG_ACTIVE; + printf(" OSD %ju peers with %zu more OSD(s): PG 1/1 state=0x%x\n", primary, peers.size(), pg.state); + bool progress = res.active; + while (progress) + { + progress = false; + for (int opcode: { BS_OP_STABLE, BS_OP_ROLLBACK }) + { + if (auto *op = f.bs->take(opcode, false)) + { + assert(op->len == 1); + apply(primary, opcode == BS_OP_STABLE, (obj_ver_id*)op->buf); + bs_complete(op, 0); + progress = true; + } + } + if (drop) + { + for (auto p: peers) + f.fail_peer_connection(p); + f.pump(); + f.complete_peering_empty(); + break; + } + for (auto p: peers) + { + std::vector ops; + for (auto & kv: f.peer(p)->sent_ops) + if (kv.second->req.hdr.opcode == OSD_OP_SEC_STABILIZE || kv.second->req.hdr.opcode == OSD_OP_SEC_ROLLBACK) + ops.push_back(kv.second); + for (auto *op: ops) + { + assert(op->req.sec_stab.len == sizeof(obj_ver_id)); + apply(p, op->req.hdr.opcode == OSD_OP_SEC_STABILIZE, (obj_ver_id*)op->iov.buf[0].iov_base); + f.peer_complete(op, 0); + progress = true; + } + } + f.pump(); + } + history = etcd_pg_history(f); + for (auto & p: parts) + printf(" OSD %ju: stable v%jx, unstable v%jx\n", p.first, p.second.stable, p.second.unstable); + return res; +} + +// After the primary (OSD 1) lost its peers, the others peer without it, then it comes back. +// Prints and returns whether the PG was active without OSD 1 and whether object A is complete in the end +static void peer_without_and_with_osd1(ec_scheme_t ec, std::map & parts, + json11::Json & history, bool & active_without, bool & complete) +{ + int pg_size = ec.data+ec.parity; + std::vector others; + for (osd_num_t osd = 3; osd <= pg_size; osd++) + others.push_back(osd); + active_without = peer_ec_all(ec, parts, 2, others, false, history).active; + others.push_back(1); + auto r = peer_ec_all(ec, parts, 2, others, false, history); + assert(r.active); + complete = !r.incomplete; + printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", complete ? "yes" : "no"); +} + +// Version 2 of object A is written to OSDs 1.. of an EC PG, unstable. OSD 1 peers, flushes it and +// loses its peers right after its own part is flushed. The others peer without it, and then it comes back. +// When OSD 1 has stabilized its part, the others must not roll version 2 back: its stable part on OSD 1 +// would then make the object incomplete. While they hold pg_data_size parts of it, the PG must be active +void test_peering_flush_primary_lost(ec_scheme_t ec, int written) +{ + printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d]\n", ec.data, ec.parity, ec.pg_minsize, written); + + std::map parts; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) }; + json11::Json history; + std::vector peers; + for (osd_num_t osd = 2; osd <= ec.data+ec.parity; osd++) + peers.push_back(osd); + peer_ec_all(ec, parts, 1, peers, true, history); + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || written == ec.data); + + printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d] passed\n", ec.data, ec.parity, ec.pg_minsize, written); +} + +// Version 2 of object A is written to OSDs 1.. of an EC PG, unstable, and a peering flush commits or +// rolls it back on all OSDs. Then OSD dies before recovery. The PG must not activate with object A +// incomplete, and must activate while the other OSDs hold pg_data_size parts of version 2 +void test_peering_flush_then_holder_lost(ec_scheme_t ec, int written, osd_num_t lost) +{ + printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost]\n", + ec.data, ec.parity, ec.pg_minsize, written, lost); + + std::map parts; + std::vector peers, others; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + { + parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) }; + if (osd != 1) + peers.push_back(osd); + if (osd != 1 && osd != lost) + others.push_back(osd); + } + json11::Json history; + peer_ec_all(ec, parts, 1, peers, false, history); + auto r = peer_ec_all(ec, parts, 1, others, false, history); + printf(" PG active without OSD %ju: %s, object incomplete: %s\n", lost, r.active ? "yes" : "no", r.incomplete ? "yes" : "no"); + assert(!r.incomplete); + assert(r.active || written-1 < ec.data); + + printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost] passed\n", + ec.data, ec.parity, ec.pg_minsize, written, lost); +} + +// Version 2 of object A is written to OSDs 1.. of a degraded EC PG, unstable, over a stable version 1 +// on all OSDs with , or as a new object. OSD 1 peers with them, flushes it and loses its peers right +// after its own part is flushed. All other OSDs peer without it, and then it comes back. The object must end +// complete or absent, never incomplete with a lone stable part +void test_peering_flush_degraded_primary_lost(ec_scheme_t ec, int written, bool older) +{ + printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s]\n", + ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object"); + + std::map parts; + std::vector peers; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + { + parts[osd] = { (uint64_t)(older ? 1 : 0), (uint64_t)(osd <= written ? 2 : 0) }; + if (osd != 1 && osd <= written) + peers.push_back(osd); + } + json11::Json history; + peer_ec_all(ec, parts, 1, peers, true, history); + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || written == ec.data); + + printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s] passed\n", + ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object"); +} + +// EC 2+1 PG 1/1 has moved from OSDs 1, 2, 3 to 1, 2, 4, and the old set is in the history. Version 2 of +// object A is unstable on OSD 1 (role 0), on OSD 3 (role 2, outside the current set) and, with , +// on OSD 2 (role 1), over a stable version 1 on OSDs 1, 2 and 3. OSD 1 peers, flushes it and loses its peers +// right after its own part is flushed. The holders of version 2, OSD 3 included, must be in the history. +// OSDs 2, 3 and 4 peer without OSD 1, and then it comes back: the object must be complete +void test_peering_flush_misplaced_primary_lost(bool on_osd2) +{ + printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3]\n", on_osd2 ? "2," : ""); + + ec_scheme_t ec = { 2, 1, 2, { 1, 2, 4 } }; + std::map parts = { + { 1, { 1, 2 } }, + { 2, { 1, (uint64_t)(on_osd2 ? 2 : 0) } }, + { 3, { 1, 2, 2 } }, + { 4, { 0, 0, 2 } }, + }; + json11::Json history = json11::Json::object { + { "osd_sets", json11::Json::array { json11::Json::array { 1, 2, 3 } } }, + }; + peer_ec_all(ec, parts, 1, { 2, 3, 4 }, true, history); + assert(parts[1].stable == 2); + assert(has_history_set(history, on_osd2 ? json11::Json::array { 1, 2, 3 } : json11::Json::array { 1, 3 })); + bool active_without = peer_ec_all(ec, parts, 2, { 3, 4 }, false, history).active; + auto r = peer_ec_all(ec, parts, 2, { 1, 3, 4 }, false, history); + printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", r.incomplete ? "no" : "yes"); + assert(r.active && !r.incomplete); + assert(active_without || !on_osd2); + + printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3] passed\n", on_osd2 ? "2," : ""); +} + +// The same for a client write to a degraded PG of pg_minsize OSDs: OSD 1 writes version 2 of object A to them +// all, stabilizes its own part and loses its peers before the others stabilize theirs. The degraded OSD set is +// in the PG history before the write +void test_degraded_write_primary_lost(ec_scheme_t ec) +{ + printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d]\n", ec.data, ec.parity, ec.pg_minsize); + + inode_t ino = INODE_WITH_POOL(1, 1); + int pg_size = ec.data+ec.parity; + std::map parts; + for (osd_num_t osd = 1; osd <= pg_size; osd++) + parts[osd] = { 1, 0 }; + json11::Json history; + { + osd_test_fixture_t f; + start_ec_all(f, ec, 1, history); + std::vector peers; + for (osd_num_t osd = 2; osd <= ec.pg_minsize; osd++) + peers.push_back(osd); + for (auto p: peers) + f.connect_peer(p); + for (auto p: peers) + f.reply_peer_list(p, { { { ino, p-1 }, 1 } }, 1); + f.reply_local_list({ { { ino, 0 }, 1 } }, 1); + f.pump(); + assert((f.pg(1, 1).state & (PG_ACTIVE|PG_DEGRADED)) == (PG_ACTIVE|PG_DEGRADED)); + auto *wr = make_write_op(ino, 0, 131072*ec.data, 0xab); + int retval = 0; + wr->callback = [&](osd_op_t *op) { retval = op->reply.hdr.retval; }; + f.exec(wr); + f.bs_zero_read_ok(1); + f.pump(); + json11::Json::array cur_set; + for (osd_num_t osd = 1; osd <= ec.pg_minsize; osd++) + cur_set.push_back(osd); + assert(has_history_set(etcd_pg_history(f), cur_set)); + auto *lw = f.bs->take(BS_OP_WRITE); + uint64_t v2 = lw->version; + bs_complete(lw, lw->len); + for (auto p: peers) + f.peer_write_ok(p, OSD_OP_SEC_WRITE, v2); + f.pump(); + parts[1].stable = v2; + for (auto p: peers) + parts[p].unstable = v2; + bs_complete(f.bs->take(BS_OP_STABLE), 0); + for (auto p: peers) + assert(f.peer_has_sent_op(p, OSD_OP_SEC_STABILIZE)); + for (auto p: peers) + f.fail_peer_connection(p); + f.pump(); + f.complete_peering_empty(); + history = etcd_pg_history(f); + delete wr; + } + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || ec.pg_minsize == ec.data); + + printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d] passed\n", ec.data, ec.parity, ec.pg_minsize); +} + // A dispatched SYNC holds its stabilize snapshot; a delete of a covered object must // wait after its fact read while the snapshot is in flight (SEC_STABILIZE of a version // the delete removes would answer -ENOENT) and must resume when the SYNC finishes. @@ -3760,6 +4108,25 @@ int main(int narg, char *args[]) test_pg_epoch_bump_above_reported_epoch(); test_chained_read_large_chain_info(); test_peering_partial_unstable_ec_rolls_back(); + test_peering_flush_primary_lost({ 2, 1, 2 }, 2); + test_peering_flush_primary_lost({ 2, 1, 2 }, 3); + test_peering_flush_primary_lost({ 3, 2, 4 }, 3); + test_peering_flush_primary_lost({ 3, 2, 4 }, 4); + test_peering_flush_primary_lost({ 3, 2, 4 }, 5); + test_degraded_write_primary_lost({ 2, 1, 2 }); + test_degraded_write_primary_lost({ 3, 2, 4 }); + test_peering_flush_then_holder_lost({ 2, 1, 2 }, 2, 2); + test_peering_flush_then_holder_lost({ 2, 1, 2 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 4, 3); + test_peering_flush_then_holder_lost({ 3, 2, 3 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 5, 3); + test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, false); + test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, true); + test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, false); + test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, true); + test_peering_flush_misplaced_primary_lost(true); + test_peering_flush_misplaced_primary_lost(false); test_delete_waits_for_stabilize_snapshot(); test_delete_waits_only_for_covered_objects(); test_replicated_add_object_to_set_degraded(); diff --git a/src/test/osd_test_fixture.h b/src/test/osd_test_fixture.h index 54af2707..bb8c891c 100644 --- a/src/test/osd_test_fixture.h +++ b/src/test/osd_test_fixture.h @@ -132,7 +132,7 @@ struct osd_test_fixture_t }); } - void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2) + void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2, int pg_minsize = 0) { auto pool_id_s = std::to_string(1); st_cli->set("/vitastor/config/pools", json11::Json::object { @@ -140,7 +140,7 @@ struct osd_test_fixture_t { "name", "pool_1" }, { "scheme", "ec" }, { "pg_size", data_chunks+parity_chunks }, - { "pg_minsize", data_chunks }, + { "pg_minsize", pg_minsize ? pg_minsize : data_chunks }, { "parity_chunks", parity_chunks }, { "pg_count", 1 }, { "failure_domain", "osd" },