From 7a95269a05772c28440e662105128ba9c30ddb96 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Mon, 16 Nov 2026 20:25:57 +0300 Subject: [PATCH 1/2] osd: Keep pg_config history in sync when a write adds the current OSD set The first write to a PG which isn't clean adds the current OSD set to pg.target_history before the new history is reported to etcd. An etcd event handled in between made apply_pg_config() compare it with the old history in pg_config and stop the PG, so the PG was re-peered for nothing. It happened on the first recovery write right after a PG state report. Update the pg_config history at the same time, like peering does when it clears the history of an active PG. --- src/osd/osd_primary_write.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index 1147bee3..0a70bf01 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -264,7 +264,11 @@ resume_3: std::sort(history_set.begin(), history_set.end()); auto it = std::lower_bound(pg.target_history.begin(), pg.target_history.end(), history_set); if (it == pg.target_history.end() || *it != history_set) + { pg.target_history.insert(it, history_set); + // Like in peering: otherwise the PG reconciling loop sees the old history and stops the PG + st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history; + } } pg.history_changed = true; report_pg_states(); -- 2.47.3 From e1f945f44393350f53333562e2a16cbcb95a2236 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Mon, 16 Nov 2026 20:25:57 +0300 Subject: [PATCH 2/2] osd: Recalculate object state online when scrub finds its target version deleted Peering left older pieces on their target OSDs out of the object state, so an incomplete EC object with its newest version on one OSD only was tracked with that piece alone. When vitastor-cli fix --check no removed it, scrub found nothing and forgot the object: reads returned zeros in place of the removed piece, and a write made them permanent, although the previous version was still readable from the other OSDs. List every OSD holding any version of the object in its state, once per role. When scrub finds no piece of the target version but outdated pieces are left, ask their OSDs for versions with SEC_READ_BMP and recalculate the object state online: the newest version becomes the target, older ones stay outdated, and pieces which are gone are dropped. The object is forgotten only if nothing is left, and without the unbalanced deref_object_state() it did before. --- .gitea/workflows/test.yml | 18 +++ src/osd/osd.h | 2 + src/osd/osd_peering_pg.cpp | 5 +- src/osd/osd_primary.h | 6 + src/osd/osd_scrub.cpp | 170 ++++++++++++++++++++++++- tests/run_tests.sh | 1 + tests/test_scrub_deleted_incomplete.sh | 69 ++++++++++ 7 files changed, 268 insertions(+), 3 deletions(-) create mode 100755 tests/test_scrub_deleted_incomplete.sh diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index e34c038c..81e7c2a2 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2183,6 +2183,24 @@ jobs: echo "" done + test_scrub_deleted_incomplete: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_scrub_deleted_incomplete.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_old_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.h b/src/osd/osd.h index fe808d2d..0e2579e0 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -344,6 +344,8 @@ class osd_t bool continue_scrub(); void submit_scrub_subops(osd_op_t *cur_op); void scrub_check_results(osd_op_t *cur_op); + bool submit_scrub_version_subops(osd_op_t *cur_op); + void scrub_recalc_state(osd_op_t *cur_op); void plan_scrub(pg_t & pg, bool report_state = true); void schedule_scrub(pg_t & pg); diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index 8ee62f93..e978041a 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -291,7 +291,8 @@ void pg_obj_state_check_t::finish_object() } if (ver_end < obj_end) { - // Check for outdated versions not present in the current target OSD set + // The state lists every OSD with any version of the object, each role on an OSD once. + // Older pieces on their target OSDs are listed too: if the target version is lost, they're all that is left for (int i = ver_end; i < obj_end; i++) { int j; @@ -307,7 +308,7 @@ void pg_obj_state_check_t::finish_object() break; } } - if (j >= osd_set.size() && pg->cur_set[list[i].oid.stripe & STRIPE_MASK] != list[i].osd_num) + if (j >= osd_set.size()) { osd_set.push_back((pg_obj_loc_t){ .role = (list[i].oid.stripe & STRIPE_MASK), diff --git a/src/osd/osd_primary.h b/src/osd/osd_primary.h index 8ee4143d..ad667765 100644 --- a/src/osd/osd_primary.h +++ b/src/osd/osd_primary.h @@ -62,6 +62,12 @@ struct osd_primary_op_data_t int chain_read_count; void *chain_info; }; + struct + { + // for scrub: parts left when no part of the target version is found + obj_ver_osd_t *scrub_parts; + int scrub_part_count; + }; }; }; diff --git a/src/osd/osd_scrub.cpp b/src/osd/osd_scrub.cpp index 112813c7..50e008c1 100644 --- a/src/osd/osd_scrub.cpp +++ b/src/osd/osd_scrub.cpp @@ -446,6 +446,43 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) } if (!total) { + pg_osd_set_state_t *object_state = NULL; + get_object_osd_set(*op_data->pg, op_data->oid, &object_state); + if (object_state != op_data->object_state) + { + // Object state changed in between by a parallel I/O operation + return; + } + int left = 0; + if (object_state) + { + for (auto & chunk: object_state->osd_set) + { + if (chunk.loc_bad & LOC_OUTDATED) + left++; + } + } + if (left > 0) + { + // All parts of the target version are deleted manually, but older ones are left: + // continue_primary_scrub() learns their versions and recalculates the object state + op_data->scrub_parts = (obj_ver_osd_t*)calloc_or_die(left, sizeof(obj_ver_osd_t)); + op_data->scrub_part_count = 0; + for (auto & chunk: object_state->osd_set) + { + if (chunk.loc_bad & LOC_OUTDATED) + { + op_data->scrub_parts[op_data->scrub_part_count++] = (obj_ver_osd_t){ + .osd_num = chunk.osd_num, + .oid = { + .inode = op_data->oid.inode, + .stripe = op_data->oid.stripe | (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role), + }, + }; + } + } + return; + } // Object is deleted manually from all OSDs, forget it printf( "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n", @@ -453,7 +490,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) op_data->oid.inode, op_data->oid.stripe ); remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false); - deref_object_state(*op_data->pg, &op_data->object_state, true); + deref_object_state(*op_data->pg, &op_data->object_state, false); return; } if (op_data->pg->scheme == POOL_SCHEME_REPLICATED) @@ -650,6 +687,121 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) }); } +// Ask OSDs for versions of the parts left in scrub_parts. Returns false if all of them are local +bool osd_t::submit_scrub_version_subops(osd_op_t *cur_op) +{ + osd_primary_op_data_t *op_data = cur_op->op_data; + int remote = 0; + for (int i = 0; i < op_data->scrub_part_count; i++) + { + obj_ver_osd_t & part = op_data->scrub_parts[i]; + if (part.osd_num == this->osd_num) + bs->read_bitmap(part.oid, UINT64_MAX, NULL, &part.version); + else + remote++; + } + op_data->done = op_data->errors = op_data->errcode = 0; + op_data->n_subops = remote; + op_data->st = 3; + if (!remote) + { + return false; + } + op_data->subops = new osd_op_t[remote]; + // The last subop may complete the operation, so op_data is not touched after it's submitted + for (int i = 0, subop_idx = 0; subop_idx < remote; i++) + { + obj_ver_osd_t *part = &op_data->scrub_parts[i]; + if (part->osd_num == this->osd_num) + continue; + osd_op_t *subop = &op_data->subops[subop_idx++]; + subop->op_type = OSD_OP_OUT; + subop->buf = malloc_or_die(sizeof(obj_ver_id)); + *(obj_ver_id*)subop->buf = (obj_ver_id){ .oid = part->oid, .version = UINT64_MAX }; + subop->req = (osd_any_op_t){ + .sec_read_bmp = { + .header = { + .magic = SECONDARY_OSD_OP_MAGIC, + .opcode = OSD_OP_SEC_READ_BMP, + }, + .len = sizeof(obj_ver_id), + } + }; + subop->callback = [this, cur_op, part](osd_op_t *subop) + { + if (subop->reply.hdr.retval == 8 + clean_entry_bitmap_size) + memcpy(&part->version, subop->buf, 8); + handle_primary_subop(subop, cur_op); + }; + submit_to_osd(subop, part->osd_num); + } + return true; +} + +// Recalculate the object state from the parts left: the newest version becomes the target, +// like in peering, older ones stay outdated, and parts which are gone are removed +void osd_t::scrub_recalc_state(osd_op_t *cur_op) +{ + osd_primary_op_data_t *op_data = cur_op->op_data; + uint64_t target_ver = 0; + for (int i = 0; i < op_data->scrub_part_count; i++) + { + if (op_data->scrub_parts[i].version > target_ver) + target_ver = op_data->scrub_parts[i].version; + } + if (!target_ver) + { + printf( + "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n", + INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe + ); + pg_osd_set_state_t *object_state = NULL; + get_object_osd_set(*op_data->pg, op_data->oid, &object_state); + if (object_state == op_data->object_state) + { + remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false); + deref_object_state(*op_data->pg, &op_data->object_state, false); + } + return; + } + printf( + "[PG %u/%u] Scrub detected deleted parts of object %jx:%jx, recalculating its state from version %ju\n", + INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe, target_ver + ); + op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false, [op_data, target_ver](pg_osd_set_t & new_set) + { + int changes = 0; + for (int i = new_set.size()-1; i >= 0; i--) + { + uint64_t version = 0; + if (new_set[i].loc_bad & LOC_OUTDATED) + { + for (int j = 0; j < op_data->scrub_part_count; j++) + { + if (op_data->scrub_parts[j].osd_num == new_set[i].osd_num && + (op_data->scrub_parts[j].oid.stripe & STRIPE_MASK) == (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : new_set[i].role)) + { + version = op_data->scrub_parts[j].version; + break; + } + } + } + if (!version) + { + // Target version parts are checked by scrub as deleted, other parts are gone + new_set.erase(new_set.begin()+i, new_set.begin()+i+1); + changes++; + } + else if (version == target_ver) + { + new_set[i].loc_bad &= ~LOC_OUTDATED; + changes++; + } + } + return changes; + }); +} + void osd_t::continue_primary_scrub(osd_op_t *cur_op) { if (!cur_op->op_data && !prepare_primary_rw(cur_op)) @@ -658,6 +810,10 @@ void osd_t::continue_primary_scrub(osd_op_t *cur_op) goto resume_1; else if (cur_op->op_data->st == 2) goto resume_2; + else if (cur_op->op_data->st == 3) + goto resume_1; + else if (cur_op->op_data->st == 4) + goto resume_4; submit_scrub_subops(cur_op); resume_1: return; @@ -677,5 +833,17 @@ resume_2: free(cur_op->op_data->stripes); cur_op->op_data->stripes = NULL; } + if (cur_op->op_data->scrub_parts && submit_scrub_version_subops(cur_op)) + { + return; + } +resume_4: + if (cur_op->op_data->scrub_parts) + { + if (!cur_op->op_data->errors) + scrub_recalc_state(cur_op); + free(cur_op->op_data->scrub_parts); + cur_op->op_data->scrub_parts = NULL; + } finish_op(cur_op, cur_op->op_data->errcode); } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index f77fb33d..dc4c6577 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -167,6 +167,7 @@ SCHEME=xor ./test_scrub.sh PG_SIZE=3 ./test_scrub.sh PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh SCHEME=ec ./test_scrub.sh +./test_scrub_deleted_incomplete.sh OLD=1 ./test_scrub.sh OLD=1 ZERO_OSD=2 ./test_scrub.sh diff --git a/tests/test_scrub_deleted_incomplete.sh b/tests/test_scrub_deleted_incomplete.sh new file mode 100755 index 00000000..4703de8c --- /dev/null +++ b/tests/test_scrub_deleted_incomplete.sh @@ -0,0 +1,69 @@ +#!/bin/bash -ex + +# Deleting the only chunk of an incomplete EC object's newest version with fix --check no +# must not make scrub forget the object: its previous version is still on the other OSDs, +# and fix must make the object state correct online, without re-peering the PG + +SCHEME=ec +IMMEDIATE_COMMIT=1 +. `dirname $0`/run_3osds.sh + +OBJ_SIZE=$((128*1024*PG_DATA_SIZE)) +LONE=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[0]') +OTHERS=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[1:][]') + +stop_others() +{ + for i in $OTHERS; do + eval kill -9 \$OSD${i}_PID + eval wait \$OSD${i}_PID || true + $ETCDCTL del /vitastor/osd/state/$i + done +} + +$VITASTOR_CLI create -s $OBJ_SIZE testimg +dd if=/dev/urandom of=./testdata/bin/a.bin bs=$OBJ_SIZE count=1 +dd if=/dev/urandom of=./testdata/bin/b.bin bs=$OBJ_SIZE count=1 + +# Version A on all OSDs, saved on all OSDs except $LONE +$VITASTOR_CLI dd if=./testdata/bin/a.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat +stop_others +for i in $OTHERS; do + cp --sparse=always ./testdata/bin/test_osd$i.bin ./testdata/bin/saved_osd$i.bin +done +for i in $OTHERS; do + start_osd $i +done +wait_up 60 + +# Version B on all OSDs, then roll the other OSDs back to version A: +# B is left only as part 0 on $LONE and the object becomes incomplete +$VITASTOR_CLI dd if=./testdata/bin/b.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat +stop_others +for i in $OTHERS; do + cp --sparse=always ./testdata/bin/saved_osd$i.bin ./testdata/bin/test_osd$i.bin + start_osd $i +done +wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -e '(.state | index(\"active\")) and (.state | index(\"has_incomplete\"))'" "PG peering with rolled back OSDs" +OBJ=$($VITASTOR_CLI describe --object-state incomplete --json | jq -r -e '.[0] | .inode+":"+.stripe') + +# Removing B leaves 4 parts of A, enough to read it +PRIMARY=$($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -r .primary) +PEERINGS=$(grep -c 'is peering' ./testdata/osd$PRIMARY.log) +$VITASTOR_CLI fix --objects $OBJ --bad-osds $LONE --part 0 --check no +$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE +if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then + format_error "Object read after fix differs from version A" +fi + +# Recovery must restore part 0 of A on $LONE +wait_up 60 +$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE +if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then + format_error "Object read after recovery differs from version A" +fi +if [[ $(grep -c 'is peering' ./testdata/osd$PRIMARY.log) != $PEERINGS ]]; then + format_error "PG was re-peered after fix" +fi + +format_green OK -- 2.47.3