From 4bdfa0185eaeed9af3ff51ac5090a4f3239e761c Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Thu, 19 Nov 2026 10:11:19 +0300 Subject: [PATCH 1/4] osd: Test a lost primary of EC objects after its own part is stabilized A primary stabilizes an EC object version by sending STABILIZE to each OSD separately, both in a peering flush and after a write with immediate_commit=all. If it completes its own part and loses its peers before theirs arrive, the next peering, without it, decides on the remaining unstable parts alone. If it rolls the version back, the stable part brought back by the primary is a lone stable part, and the object stays incomplete. Run this for a peering flush of a version written to some or all OSDs of EC 2+1 with pg_minsize 2 and EC 3+2 with pg_minsize 4, and for a write to a degraded PG of pg_minsize OSDs. The object must be complete when the primary returns, and the PG must stay active while the primary is away unless the version reached exactly pg_data_size OSDs. Also lose one holder after a completed flush, before recovery: the PG must not activate with the object incomplete, and must activate while the others hold pg_data_size parts of the version. The current rule (n_roles >= pg_size) fails EC 2+1 with the version on all 3 OSDs, EC 3+2 with the version on all 5, and the degraded write to 4 OSDs of EC 3+2. --- src/test/osd_test.cpp | 290 ++++++++++++++++++++++++++++++++++++ src/test/osd_test_fixture.h | 4 +- 2 files changed, 292 insertions(+), 2 deletions(-) diff --git a/src/test/osd_test.cpp b/src/test/osd_test.cpp index 63633d08..4b13f9bf 100644 --- a/src/test/osd_test.cpp +++ b/src/test/osd_test.cpp @@ -3247,6 +3247,283 @@ void test_peering_partial_unstable_ec_rolls_back() printf("test_peering_partial_unstable_ec_rolls_back passed\n"); } +// Complete a captured blockstore op like the store does: through a copy of its +// callback, because some callbacks delete the op that holds them +static void bs_complete(blockstore_op_t *op, int retval) +{ + op->retval = retval; + std::function(op->callback)(op); +} + +// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none +struct ec_part_t +{ + uint64_t stable, unstable; +}; + +// EC + pool with , PG 1/1 on OSDs 1..pg_size, OSD n holds role n-1 +struct ec_scheme_t +{ + int data, parity, pg_minsize; +}; + +struct ec_peering_t +{ + bool active = false, incomplete = false; +}; + +static void start_ec_all(osd_test_fixture_t & f, ec_scheme_t ec, osd_num_t primary, const json11::Json & history) +{ + f.configure_ec_pool(ec.data, ec.parity, primary, ec.pg_minsize); + if (!history.is_null()) + f.st_cli->set("/vitastor/pg/history/1/1", history); + f.start(json11::Json::object { + { "osd_num", primary }, + { "etcd_address", "127.0.0.1:2379" }, + { "immediate_commit", "all" }, + { "block_size", 131072 }, + { "bitmap_granularity", 4096 }, + }); +} + +static json11::Json etcd_pg_history(osd_test_fixture_t & f) +{ + auto hist_it = f.etcd_store.data.find("/vitastor/pg/history/1/1"); + if (hist_it == f.etcd_store.data.end()) + return json11::Json(); + std::string json_err; + return json11::Json::parse(hist_it->second.value, json_err); +} + +static bool has_history_set(const json11::Json & history, json11::Json::array set) +{ + auto & sets = history["osd_sets"].array_items(); + return std::find(sets.begin(), sets.end(), json11::Json(set)) != sets.end(); +} + +// One peering of PG 1/1 by with up and object A as in , immediate_commit=all. +// Applies the stabilize and rollback ops peering sends. With , the primary loses its peers right +// after its own op completes, so the peers keep their parts. is the PG history in etcd, +// carried between peerings +static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map & parts, osd_num_t primary, + std::vector peers, bool drop, json11::Json & history) +{ + inode_t ino = INODE_WITH_POOL(1, 1); + ec_peering_t res; + auto list = [&](osd_num_t osd) + { + std::vector l; + object_id oid = { .inode = ino, .stripe = osd-1 }; + if (parts[osd].stable) + l.push_back({ oid, parts[osd].stable }); + if (parts[osd].unstable) + l.push_back({ oid, parts[osd].unstable }); + return l; + }; + auto apply = [&](osd_num_t osd, bool stabilize, obj_ver_id *ov) + { + assert(ov->oid.inode == ino && ov->oid.stripe == osd-1); + if (stabilize) + { + assert(ov->version == parts[osd].unstable); + parts[osd].stable = parts[osd].unstable; + } + else + assert(ov->version == parts[osd].stable); + parts[osd].unstable = 0; + }; + osd_test_fixture_t f; + start_ec_all(f, ec, primary, history); + for (auto p: peers) + f.connect_peer(p); + auto & pg = f.pg(1, 1); + if (pg.state == PG_PEERING) + { + for (auto p: peers) + f.reply_peer_list(p, list(p), parts[p].stable ? 1 : 0); + f.reply_local_list(list(primary), parts[primary].stable ? 1 : 0); + f.pump(); + } + res.incomplete = pg.state & PG_HAS_INCOMPLETE; + res.active = pg.state & PG_ACTIVE; + printf(" OSD %ju peers with %zu more OSD(s): PG 1/1 state=0x%x\n", primary, peers.size(), pg.state); + bool progress = res.active; + while (progress) + { + progress = false; + for (int opcode: { BS_OP_STABLE, BS_OP_ROLLBACK }) + { + if (auto *op = f.bs->take(opcode, false)) + { + assert(op->len == 1); + apply(primary, opcode == BS_OP_STABLE, (obj_ver_id*)op->buf); + bs_complete(op, 0); + progress = true; + } + } + if (drop) + { + for (auto p: peers) + f.fail_peer_connection(p); + f.pump(); + f.complete_peering_empty(); + break; + } + for (auto p: peers) + { + std::vector ops; + for (auto & kv: f.peer(p)->sent_ops) + if (kv.second->req.hdr.opcode == OSD_OP_SEC_STABILIZE || kv.second->req.hdr.opcode == OSD_OP_SEC_ROLLBACK) + ops.push_back(kv.second); + for (auto *op: ops) + { + assert(op->req.sec_stab.len == sizeof(obj_ver_id)); + apply(p, op->req.hdr.opcode == OSD_OP_SEC_STABILIZE, (obj_ver_id*)op->iov.buf[0].iov_base); + f.peer_complete(op, 0); + progress = true; + } + } + f.pump(); + } + history = etcd_pg_history(f); + for (auto & p: parts) + printf(" OSD %ju: stable v%jx, unstable v%jx\n", p.first, p.second.stable, p.second.unstable); + return res; +} + +// After the primary (OSD 1) lost its peers, the others peer without it, then it comes back. +// Prints and returns whether the PG was active without OSD 1 and whether object A is complete in the end +static void peer_without_and_with_osd1(ec_scheme_t ec, std::map & parts, + json11::Json & history, bool & active_without, bool & complete) +{ + int pg_size = ec.data+ec.parity; + std::vector others; + for (osd_num_t osd = 3; osd <= pg_size; osd++) + others.push_back(osd); + active_without = peer_ec_all(ec, parts, 2, others, false, history).active; + others.push_back(1); + auto r = peer_ec_all(ec, parts, 2, others, false, history); + assert(r.active); + complete = !r.incomplete; + printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", complete ? "yes" : "no"); +} + +// Version 2 of object A is written to OSDs 1.. of an EC PG, unstable. OSD 1 peers, flushes it and +// loses its peers right after its own part is flushed. The others peer without it, and then it comes back. +// When OSD 1 has stabilized its part, the others must not roll version 2 back: its stable part on OSD 1 +// would then make the object incomplete. While they hold pg_data_size parts of it, the PG must be active +void test_peering_flush_primary_lost(ec_scheme_t ec, int written) +{ + printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d]\n", ec.data, ec.parity, ec.pg_minsize, written); + + std::map parts; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) }; + json11::Json history; + std::vector peers; + for (osd_num_t osd = 2; osd <= ec.data+ec.parity; osd++) + peers.push_back(osd); + peer_ec_all(ec, parts, 1, peers, true, history); + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || written == ec.data); + + printf("test_peering_flush_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d] passed\n", ec.data, ec.parity, ec.pg_minsize, written); +} + +// Version 2 of object A is written to OSDs 1.. of an EC PG, unstable, and a peering flush commits or +// rolls it back on all OSDs. Then OSD dies before recovery. The PG must not activate with object A +// incomplete, and must activate while the other OSDs hold pg_data_size parts of version 2 +void test_peering_flush_then_holder_lost(ec_scheme_t ec, int written, osd_num_t lost) +{ + printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost]\n", + ec.data, ec.parity, ec.pg_minsize, written, lost); + + std::map parts; + std::vector peers, others; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + { + parts[osd] = { 1, (uint64_t)(osd <= written ? 2 : 0) }; + if (osd != 1) + peers.push_back(osd); + if (osd != 1 && osd != lost) + others.push_back(osd); + } + json11::Json history; + peer_ec_all(ec, parts, 1, peers, false, history); + auto r = peer_ec_all(ec, parts, 1, others, false, history); + printf(" PG active without OSD %ju: %s, object incomplete: %s\n", lost, r.active ? "yes" : "no", r.incomplete ? "yes" : "no"); + assert(!r.incomplete); + assert(r.active || written-1 < ec.data); + + printf("test_peering_flush_then_holder_lost[EC %d+%d, pg_minsize %d, v2 on %d, OSD %ju lost] passed\n", + ec.data, ec.parity, ec.pg_minsize, written, lost); +} + +// The same for a client write to a degraded PG of pg_minsize OSDs: OSD 1 writes version 2 of object A to them +// all, stabilizes its own part and loses its peers before the others stabilize theirs. The degraded OSD set is +// in the PG history before the write +void test_degraded_write_primary_lost(ec_scheme_t ec) +{ + printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d]\n", ec.data, ec.parity, ec.pg_minsize); + + inode_t ino = INODE_WITH_POOL(1, 1); + int pg_size = ec.data+ec.parity; + std::map parts; + for (osd_num_t osd = 1; osd <= pg_size; osd++) + parts[osd] = { 1, 0 }; + json11::Json history; + { + osd_test_fixture_t f; + start_ec_all(f, ec, 1, history); + std::vector peers; + for (osd_num_t osd = 2; osd <= ec.pg_minsize; osd++) + peers.push_back(osd); + for (auto p: peers) + f.connect_peer(p); + for (auto p: peers) + f.reply_peer_list(p, { { { ino, p-1 }, 1 } }, 1); + f.reply_local_list({ { { ino, 0 }, 1 } }, 1); + f.pump(); + assert((f.pg(1, 1).state & (PG_ACTIVE|PG_DEGRADED)) == (PG_ACTIVE|PG_DEGRADED)); + auto *wr = make_write_op(ino, 0, 131072*ec.data, 0xab); + int retval = 0; + wr->callback = [&](osd_op_t *op) { retval = op->reply.hdr.retval; }; + f.exec(wr); + f.bs_zero_read_ok(1); + f.pump(); + json11::Json::array cur_set; + for (osd_num_t osd = 1; osd <= ec.pg_minsize; osd++) + cur_set.push_back(osd); + assert(has_history_set(etcd_pg_history(f), cur_set)); + auto *lw = f.bs->take(BS_OP_WRITE); + uint64_t v2 = lw->version; + bs_complete(lw, lw->len); + for (auto p: peers) + f.peer_write_ok(p, OSD_OP_SEC_WRITE, v2); + f.pump(); + parts[1].stable = v2; + for (auto p: peers) + parts[p].unstable = v2; + bs_complete(f.bs->take(BS_OP_STABLE), 0); + for (auto p: peers) + assert(f.peer_has_sent_op(p, OSD_OP_SEC_STABILIZE)); + for (auto p: peers) + f.fail_peer_connection(p); + f.pump(); + f.complete_peering_empty(); + history = etcd_pg_history(f); + delete wr; + } + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || ec.pg_minsize == ec.data); + + printf("test_degraded_write_primary_lost[EC %d+%d, pg_minsize %d] passed\n", ec.data, ec.parity, ec.pg_minsize); +} + // A dispatched SYNC holds its stabilize snapshot; a delete of a covered object must // wait after its fact read while the snapshot is in flight (SEC_STABILIZE of a version // the delete removes would answer -ENOENT) and must resume when the SYNC finishes. @@ -3760,6 +4037,19 @@ int main(int narg, char *args[]) test_pg_epoch_bump_above_reported_epoch(); test_chained_read_large_chain_info(); test_peering_partial_unstable_ec_rolls_back(); + test_peering_flush_primary_lost({ 2, 1, 2 }, 2); + test_peering_flush_primary_lost({ 2, 1, 2 }, 3); + test_peering_flush_primary_lost({ 3, 2, 4 }, 3); + test_peering_flush_primary_lost({ 3, 2, 4 }, 4); + test_peering_flush_primary_lost({ 3, 2, 4 }, 5); + test_degraded_write_primary_lost({ 2, 1, 2 }); + test_degraded_write_primary_lost({ 3, 2, 4 }); + test_peering_flush_then_holder_lost({ 2, 1, 2 }, 2, 2); + test_peering_flush_then_holder_lost({ 2, 1, 2 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 4, 3); + test_peering_flush_then_holder_lost({ 3, 2, 3 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 3, 2); + test_peering_flush_then_holder_lost({ 3, 2, 4 }, 5, 3); test_delete_waits_for_stabilize_snapshot(); test_delete_waits_only_for_covered_objects(); test_replicated_add_object_to_set_degraded(); diff --git a/src/test/osd_test_fixture.h b/src/test/osd_test_fixture.h index 54af2707..bb8c891c 100644 --- a/src/test/osd_test_fixture.h +++ b/src/test/osd_test_fixture.h @@ -132,7 +132,7 @@ struct osd_test_fixture_t }); } - void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2) + void configure_ec_pool(int data_chunks, int parity_chunks, uint64_t primary_osd = 2, int pg_minsize = 0) { auto pool_id_s = std::to_string(1); st_cli->set("/vitastor/config/pools", json11::Json::object { @@ -140,7 +140,7 @@ struct osd_test_fixture_t { "name", "pool_1" }, { "scheme", "ec" }, { "pg_size", data_chunks+parity_chunks }, - { "pg_minsize", data_chunks }, + { "pg_minsize", pg_minsize ? pg_minsize : data_chunks }, { "parity_chunks", parity_chunks }, { "pg_count", 1 }, { "failure_domain", "osd" }, -- 2.47.3 From 7a099eaa505cb1f48c830db2aa2736643b3a26e6 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Thu, 19 Nov 2026 12:50:23 +0300 Subject: [PATCH 2/4] osd: Put the holders of unstable target versions into the PG history A flush stabilizes the pieces of an unstable target version one OSD at a time. If the primary stabilizes its own piece and dies, the others see the version on fewer roles and may roll it back, leaving a lone stable piece on the primary. When the flush is going to stabilize the target version of an object, add the set of OSDs holding it to the PG history, and make every flush batch wait until the history is reported with the epoch, like writes do. A later peering then refuses to activate the PG while fewer than pg_data_size of them are up, so the version is never decided on without enough of its holders. A batch waiting for the report holds a repeer or a stop of the PG like a write does, and is dropped if the PG is no longer active by then. --- src/osd/osd.h | 1 + src/osd/osd_flush.cpp | 17 +++++++++++++++++ src/osd/osd_peering.cpp | 7 +++++++ src/osd/osd_peering_pg.cpp | 19 +++++++++++++++++++ src/osd/osd_peering_pg.h | 1 + src/osd/osd_primary_write.cpp | 13 +++++++++++++ 6 files changed, 58 insertions(+) diff --git a/src/osd/osd.h b/src/osd/osd.h index 0e2579e0..b146565c 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -329,6 +329,7 @@ class osd_t // flushing, recovery and backfill void submit_pg_flush_ops(pg_t & pg); + void submit_flush_batch(pg_t & pg); void handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval); bool submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t peer_osd, int count, obj_ver_id *data); bool pick_next_recovery(osd_recovery_op_t &op); diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 96738cd1..402ba0a4 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -45,6 +45,23 @@ void osd_t::submit_pg_flush_ops(pg_t & pg) first = false; it++; } + if (pg.epoch > pg.reported_epoch) + { + // Flushes wait until the history with the holders of their versions is reported + if (pg.epoch > pg.reporting_epoch) + { + this->pg_state_dirty.insert({ .pool_id = pg.pool_id, .pg_num = pg.pg_num }); + pg.history_changed = true; + report_pg_states(); + } + return; + } + submit_flush_batch(pg); +} + +void osd_t::submit_flush_batch(pg_t & pg) +{ + pg_flush_batch_t *fb = pg.flush_batch; for (auto & l: fb->rollback_lists) { if (l.second.size() > 0) diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index cdffbcf1..1a88da05 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -354,7 +354,14 @@ bool osd_t::continue_pg_peering(pg_t & pg) } if (pg.peering_state->lists_done) { + auto history_size = pg.target_history.size(); pg.calc_object_states(log_level); + if (pg.target_history.size() != history_size) + { + // Reported with the epoch before the first write or flush. Like there: otherwise + // the PG reconciling loop sees the old history and stops the PG + st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history; + } report_pg_state(pg); schedule_scrub(pg); inconsistent_objects += pg.inconsistent_objects.size(); diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index e978041a..e4cd3bb1 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -200,6 +200,7 @@ void pg_obj_state_check_t::finish_object() { pg->state |= PG_HAS_UNCLEAN; std::unordered_map pieces; + bool stabilize_target = false; for (int i = obj_start; i < obj_end; i++) { auto & pcs = pieces[(obj_piece_id_t){ .oid = list[i].oid, .osd_num = list[i].osd_num }]; @@ -232,9 +233,27 @@ void pg_obj_state_check_t::finish_object() { act.make_stable = true; act.stable_to = pcs.max_ver > target_ver ? pcs.max_target : pcs.max_ver; + stabilize_target = stabilize_target || act.stable_to == target_ver; } } } + if (stabilize_target) + { + // The flush stabilizes the target version one OSD at a time. Put the OSDs holding + // it into the history, so that a peering without some of them doesn't roll it back + std::vector holders; + for (int i = ver_start; i < ver_end; i++) + { + holders.push_back(list[i].osd_num); + } + std::sort(holders.begin(), holders.end()); + holders.erase(std::unique(holders.begin(), holders.end()), holders.end()); + auto it = std::lower_bound(pg->target_history.begin(), pg->target_history.end(), holders); + if (it == pg->target_history.end() || *it != holders) + { + pg->target_history.insert(it, holders); + } + } } if (!target_ver) { diff --git a/src/osd/osd_peering_pg.h b/src/osd/osd_peering_pg.h index 3bcc2fb6..cdbd91ef 100644 --- a/src/osd/osd_peering_pg.h +++ b/src/osd/osd_peering_pg.h @@ -79,6 +79,7 @@ struct pg_flush_batch_t { std::map> rollback_lists; std::map> stable_lists; + // 0 while the batch waits for its epoch to be reported int flush_ops = 0, flush_done = 0, flush_errors = 0; int flush_objects = 0; }; diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index 0a70bf01..fa5dd390 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -545,6 +545,19 @@ void osd_t::on_change_pg_history_hook(pool_id_t pool_id, pg_num_t pg_num) } } } + pg_it = pgs.find({ .pool_id = pool_id, .pg_num = pg_num }); + if (pg_it != pgs.end() && pg_it->second.flush_batch && !pg_it->second.flush_batch->flush_ops) + { + // A flush batch waits for the epoch too, and holds a repeer or a stop like a write + if (pg_it->second.state & PG_ACTIVE) + submit_flush_batch(pg_it->second); + else + { + delete pg_it->second.flush_batch; + pg_it->second.flush_batch = NULL; + continue_pg(pg_it->second); + } + } } } -- 2.47.3 From 3ce7aea3cbff6b10a2fd9be32df7cc8f7916e17e Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Thu, 19 Nov 2026 10:13:14 +0300 Subject: [PATCH 3/4] osd: Accept unstable EC versions on peering from pg_data_size roles again Since 368daa4b, peering targets an unstable EC version only if it is on all pg_size roles, while the history check lets a PG activate with pg_data_size OSDs of a history set. With a higher commit threshold than that, a peering without some holders of a version rolls it back even if it is already stabilized on them, and the stable part they bring back leaves the object incomplete. A primary that stabilizes its own part of a version written to all OSDs and dies is enough for that. Target an unstable version from pg_data_size roles again, so the commit threshold equals the history check. The holders of unstable target versions are in the PG history now, so instead of committing a version with no redundancy left after losing a holder, the PG stays inactive until enough of them are back. Update the 368daa4b test: a partial chain on pg_data_size roles is now committed and stabilized, and its holders are in the history. --- src/osd/osd_peering_pg.cpp | 14 +++++------ src/test/osd_test.cpp | 51 +++++++++++++++++++------------------- 2 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index e4cd3bb1..0cab0751 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -110,11 +110,11 @@ void pg_obj_state_check_t::start_object() void pg_obj_state_check_t::handle_version() { - // A version becomes target only if it's stable (successfully written to the previous set) - // or unstable but full. We could target any recoverable version, but in this case some - // client's writes written to OSD set A may actually land on a smaller subset of A. - // And then client can't be sure that the pg_minsize guarantee actually holds. - if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_size)) + // A version becomes target if it's stable (possibly acked) or unstable but recoverable. + // A higher threshold than the history check (pg_data_size) would let a peering without + // some holders roll back a version already stabilized on them; holders of unstable + // target versions go to the history, so such a peering doesn't activate instead + if (!target_ver && last_ver != list[list_pos].version && (n_stable > 0 || n_roles >= pg->pg_data_size)) { target_ver = last_ver; ver_end = list_pos; @@ -178,9 +178,9 @@ void pg_obj_state_check_t::handle_version() void pg_obj_state_check_t::finish_object() { - if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_size)) + if (!target_ver && (n_stable > 0 || n_roles >= pg->pg_data_size)) { - // Stable (possibly acked) or fully materialized unstable - see handle_version() + // Stable (possibly acked) or recoverable unstable - see handle_version() target_ver = last_ver; ver_end = list_pos; } diff --git a/src/test/osd_test.cpp b/src/test/osd_test.cpp index 4b13f9bf..4e5ae4cd 100644 --- a/src/test/osd_test.cpp +++ b/src/test/osd_test.cpp @@ -3107,16 +3107,14 @@ void test_pg_epoch_bump_above_reported_epoch() printf("test_pg_epoch_bump_above_reported_epoch passed\n"); } -// A partial unstable chain left by a failed/rejected EC write must not become the -// committed version even when it holds pg_data_size pieces: that would turn a -// possibly-nonexistent write into committed data with zero redundancy. Peering must -// fall back to the older stable version and roll the fragments back. A stable -// fragment, on the other hand, is possibly-acked - the OSD can never know otherwise -// - so it stays the target and is never rolled back, incomplete or not. -// A partial chain of pg_data_size pieces is not committed either, even when a -// degraded set holds a fragment per member: an EC 2+2 pair on the two data roles -// of a 2-member set still cannot decode (subcase D: refused-write fragments of a -// fresh object - peering must delete them, not commit). +// A partial unstable chain left by a failed/rejected EC write becomes the committed +// version when it holds pg_data_size pieces: a flush may have stabilized some of them +// already, and the history check lets a later peering activate with pg_data_size OSDs +// of a set, so a higher threshold would roll back a version stable on missing OSDs. +// The holders of the chain go to the PG history instead, so a peering without +// pg_data_size of them doesn't activate. A stable fragment is possibly-acked - the OSD +// can never know otherwise - so it stays the target and is never rolled back, +// incomplete or not. // // Layout: EC 2+2, PG 1/1, cur_set = role 0..3 => OSD 1..4. Object INODE_WITH_POOL(1,1) // is stable at v1 on all roles; a rejected write left v2 unstable on role 0 (OSD 1) @@ -3157,7 +3155,7 @@ void test_peering_partial_unstable_ec_rolls_back() }; }; - // ============ A: partial unstable v2 falls back to stable v1 ============ + // ============ A: partial unstable v2 on pg_data_size roles is committed ============ pg_t *pg = make_pg(); add_list(pg, 1, { { 0, 1 }, { 0, 2 } }, 1); @@ -3166,18 +3164,21 @@ void test_peering_partial_unstable_ec_rolls_back() add_list(pg, 4, { { 3, 1 } }, 1); pg->calc_object_states(0); - // v1 is the target: the object is clean at v1, reads see v1, fragments roll back - assert(pg->total_count == 1 && pg->clean_count == 1); - assert(!(pg->state & (PG_HAS_INCOMPLETE|PG_HAS_DEGRADED))); + // v2 is the target: the object is degraded at v2, the fragments are stabilized, and their + // holders are in the history, so losing one before the PG is clean keeps the PG inactive + assert(pg->total_count == 1 && pg->clean_count == 0); + assert(pg->state & PG_HAS_DEGRADED); + assert(!(pg->state & PG_HAS_INCOMPLETE)); assert(pg->state & PG_HAS_UNCLEAN); - assert(!find_object_state(*pg, oid)); - assert(pg->ver_override.count(oid) && pg->ver_override[oid] == 1); + assert(find_object_state(*pg, oid)); + assert(!pg->ver_override.count(oid)); assert(pg->flush_actions.size() == 2); for (auto & fa: pg->flush_actions) { - assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 1); + assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2); assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 2u)); } + assert(pg->target_history == std::vector>({ { 1, 3 } })); delete pg->peering_state; delete pg; @@ -3216,11 +3217,10 @@ void test_peering_partial_unstable_ec_rolls_back() delete pg; // ============ D: an unstable chain that was never whole. v2 exists on roles 0 and 1 - // only - pg_data_size pieces, but EC 2+2 needs all 4 to decode, and the object never - // existed at any stable version: this is what a refused write to a fresh object - // leaves behind (seed 1996). The old rule (n_roles >= pg_data_size) targeted such a - // chain and peering-flush committed it; the new rule must refuse v2 and delete the - // fragments entirely ============ + // only - pg_data_size pieces of a degraded 2-member set, and the object never existed + // at any stable version: this is what a refused write to a fresh object leaves behind + // (seed 1996). The pieces are recoverable and may be stable somewhere already, so v2 + // is committed and the object exists ============ pg = make_pg(); pg->pg_cursize = 2; @@ -3230,17 +3230,18 @@ void test_peering_partial_unstable_ec_rolls_back() add_list(pg, 2, { { 1, 2 } }, 0); pg->calc_object_states(0); - // Nothing is committed: the object stays absent, both fragments are deleted - assert(pg->total_count == 0 && pg->clean_count == 0); + // v2 is committed: the object exists, both fragments are stabilized, their holders are in the history + assert(pg->total_count == 1 && pg->clean_count == 1); assert(pg->state & PG_HAS_UNCLEAN); assert(!(pg->state & PG_HAS_INCOMPLETE)); assert(!pg->ver_override.count(oid)); assert(pg->flush_actions.size() == 2); for (auto & fa: pg->flush_actions) { - assert(fa.second.rollback && !fa.second.make_stable && fa.second.rollback_to == 0); + assert(fa.second.make_stable && !fa.second.rollback && fa.second.stable_to == 2); assert((fa.first.oid.stripe & STRIPE_MASK) == (fa.first.osd_num == 1 ? 0u : 1u)); } + assert(pg->target_history == std::vector>({ { 1, 2 } })); delete pg->peering_state; delete pg; -- 2.47.3 From c038bf68045a1c0e1d13f839a43595621a0fe559 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Thu, 19 Nov 2026 10:21:44 +0300 Subject: [PATCH 4/4] osd: Test lost primaries of degraded and misplaced EC objects Run the lost primary scenario for a version written to all OSDs of a degraded PG, EC 2+2 with one OSD down and EC 3+2 with pg_minsize 4 and one OSD down, both over an older stable version and as a new object with no pieces on the missing OSD. The object must end complete or absent, never incomplete with a lone stable part. Also run it for a misplaced object: the PG has moved from OSDs 1, 2, 3 to 1, 2, 4 and the unstable version has a piece on OSD 3, outside the current set. Its holders, OSD 3 included, must go to the history, and the object must be complete when the primary returns. Let the EC test helpers take an OSD set and a piece role per OSD. --- src/test/osd_test.cpp | 84 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 80 insertions(+), 4 deletions(-) diff --git a/src/test/osd_test.cpp b/src/test/osd_test.cpp index 4e5ae4cd..1df6133a 100644 --- a/src/test/osd_test.cpp +++ b/src/test/osd_test.cpp @@ -3256,16 +3256,18 @@ static void bs_complete(blockstore_op_t *op, int retval) std::function(op->callback)(op); } -// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none +// Object A of an EC PG on one OSD: its stable and unstable versions, 0 if none, and its role if not osd-1 struct ec_part_t { uint64_t stable, unstable; + int role = -1; }; -// EC + pool with , PG 1/1 on OSDs 1..pg_size, OSD n holds role n-1 +// EC + pool with , PG 1/1 on or OSDs 1..pg_size, OSD n holds role n-1 struct ec_scheme_t { int data, parity, pg_minsize; + std::vector osd_set = {}; }; struct ec_peering_t @@ -3276,6 +3278,15 @@ struct ec_peering_t static void start_ec_all(osd_test_fixture_t & f, ec_scheme_t ec, osd_num_t primary, const json11::Json & history) { f.configure_ec_pool(ec.data, ec.parity, primary, ec.pg_minsize); + if (ec.osd_set.size()) + { + f.st_cli->set("/vitastor/pg/config", json11::Json::object { + { "items", json11::Json::object { { "1", json11::Json::object { { "1", json11::Json::object { + { "osd_set", ec.osd_set }, + { "primary", primary }, + } } } } } }, + }); + } if (!history.is_null()) f.st_cli->set("/vitastor/pg/history/1/1", history); f.start(json11::Json::object { @@ -3314,7 +3325,7 @@ static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map & auto list = [&](osd_num_t osd) { std::vector l; - object_id oid = { .inode = ino, .stripe = osd-1 }; + object_id oid = { .inode = ino, .stripe = (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1) }; if (parts[osd].stable) l.push_back({ oid, parts[osd].stable }); if (parts[osd].unstable) @@ -3323,7 +3334,7 @@ static ec_peering_t peer_ec_all(ec_scheme_t ec, std::map & }; auto apply = [&](osd_num_t osd, bool stabilize, obj_ver_id *ov) { - assert(ov->oid.inode == ino && ov->oid.stripe == osd-1); + assert(ov->oid.inode == ino && ov->oid.stripe == (uint64_t)(parts[osd].role >= 0 ? parts[osd].role : osd-1)); if (stabilize) { assert(ov->version == parts[osd].unstable); @@ -3462,6 +3473,65 @@ void test_peering_flush_then_holder_lost(ec_scheme_t ec, int written, osd_num_t ec.data, ec.parity, ec.pg_minsize, written, lost); } +// Version 2 of object A is written to OSDs 1.. of a degraded EC PG, unstable, over a stable version 1 +// on all OSDs with , or as a new object. OSD 1 peers with them, flushes it and loses its peers right +// after its own part is flushed. All other OSDs peer without it, and then it comes back. The object must end +// complete or absent, never incomplete with a lone stable part +void test_peering_flush_degraded_primary_lost(ec_scheme_t ec, int written, bool older) +{ + printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s]\n", + ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object"); + + std::map parts; + std::vector peers; + for (osd_num_t osd = 1; osd <= ec.data+ec.parity; osd++) + { + parts[osd] = { (uint64_t)(older ? 1 : 0), (uint64_t)(osd <= written ? 2 : 0) }; + if (osd != 1 && osd <= written) + peers.push_back(osd); + } + json11::Json history; + peer_ec_all(ec, parts, 1, peers, true, history); + bool active_without = false, complete = false; + peer_without_and_with_osd1(ec, parts, history, active_without, complete); + assert(complete); + assert(active_without || written == ec.data); + + printf("test_peering_flush_degraded_primary_lost[EC %d+%d, pg_minsize %d, v2 on %d, %s] passed\n", + ec.data, ec.parity, ec.pg_minsize, written, older ? "over v1" : "new object"); +} + +// EC 2+1 PG 1/1 has moved from OSDs 1, 2, 3 to 1, 2, 4, and the old set is in the history. Version 2 of +// object A is unstable on OSD 1 (role 0), on OSD 3 (role 2, outside the current set) and, with , +// on OSD 2 (role 1), over a stable version 1 on OSDs 1, 2 and 3. OSD 1 peers, flushes it and loses its peers +// right after its own part is flushed. The holders of version 2, OSD 3 included, must be in the history. +// OSDs 2, 3 and 4 peer without OSD 1, and then it comes back: the object must be complete +void test_peering_flush_misplaced_primary_lost(bool on_osd2) +{ + printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3]\n", on_osd2 ? "2," : ""); + + ec_scheme_t ec = { 2, 1, 2, { 1, 2, 4 } }; + std::map parts = { + { 1, { 1, 2 } }, + { 2, { 1, (uint64_t)(on_osd2 ? 2 : 0) } }, + { 3, { 1, 2, 2 } }, + { 4, { 0, 0, 2 } }, + }; + json11::Json history = json11::Json::object { + { "osd_sets", json11::Json::array { json11::Json::array { 1, 2, 3 } } }, + }; + peer_ec_all(ec, parts, 1, { 2, 3, 4 }, true, history); + assert(parts[1].stable == 2); + assert(has_history_set(history, on_osd2 ? json11::Json::array { 1, 2, 3 } : json11::Json::array { 1, 3 })); + bool active_without = peer_ec_all(ec, parts, 2, { 3, 4 }, false, history).active; + auto r = peer_ec_all(ec, parts, 2, { 1, 3, 4 }, false, history); + printf(" PG active without OSD 1: %s, object complete: %s\n", active_without ? "yes" : "no", r.incomplete ? "no" : "yes"); + assert(r.active && !r.incomplete); + assert(active_without || !on_osd2); + + printf("test_peering_flush_misplaced_primary_lost[v2 on OSDs 1,%s3] passed\n", on_osd2 ? "2," : ""); +} + // The same for a client write to a degraded PG of pg_minsize OSDs: OSD 1 writes version 2 of object A to them // all, stabilizes its own part and loses its peers before the others stabilize theirs. The degraded OSD set is // in the PG history before the write @@ -4051,6 +4121,12 @@ int main(int narg, char *args[]) test_peering_flush_then_holder_lost({ 3, 2, 3 }, 3, 2); test_peering_flush_then_holder_lost({ 3, 2, 4 }, 3, 2); test_peering_flush_then_holder_lost({ 3, 2, 4 }, 5, 3); + test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, false); + test_peering_flush_degraded_primary_lost({ 2, 2, 3 }, 3, true); + test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, false); + test_peering_flush_degraded_primary_lost({ 3, 2, 4 }, 4, true); + test_peering_flush_misplaced_primary_lost(true); + test_peering_flush_misplaced_primary_lost(false); test_delete_waits_for_stabilize_snapshot(); test_delete_waits_only_for_covered_objects(); test_replicated_add_object_to_set_degraded(); -- 2.47.3