diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index e34c038c..81e7c2a2 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2183,6 +2183,24 @@ jobs: echo "" done + test_scrub_deleted_incomplete: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_scrub_deleted_incomplete.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_old_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.h b/src/osd/osd.h index fe808d2d..0e2579e0 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -344,6 +344,8 @@ class osd_t bool continue_scrub(); void submit_scrub_subops(osd_op_t *cur_op); void scrub_check_results(osd_op_t *cur_op); + bool submit_scrub_version_subops(osd_op_t *cur_op); + void scrub_recalc_state(osd_op_t *cur_op); void plan_scrub(pg_t & pg, bool report_state = true); void schedule_scrub(pg_t & pg); diff --git a/src/osd/osd_peering_pg.cpp b/src/osd/osd_peering_pg.cpp index 8ee62f93..e978041a 100644 --- a/src/osd/osd_peering_pg.cpp +++ b/src/osd/osd_peering_pg.cpp @@ -291,7 +291,8 @@ void pg_obj_state_check_t::finish_object() } if (ver_end < obj_end) { - // Check for outdated versions not present in the current target OSD set + // The state lists every OSD with any version of the object, each role on an OSD once. + // Older pieces on their target OSDs are listed too: if the target version is lost, they're all that is left for (int i = ver_end; i < obj_end; i++) { int j; @@ -307,7 +308,7 @@ void pg_obj_state_check_t::finish_object() break; } } - if (j >= osd_set.size() && pg->cur_set[list[i].oid.stripe & STRIPE_MASK] != list[i].osd_num) + if (j >= osd_set.size()) { osd_set.push_back((pg_obj_loc_t){ .role = (list[i].oid.stripe & STRIPE_MASK), diff --git a/src/osd/osd_primary.h b/src/osd/osd_primary.h index 8ee4143d..ad667765 100644 --- a/src/osd/osd_primary.h +++ b/src/osd/osd_primary.h @@ -62,6 +62,12 @@ struct osd_primary_op_data_t int chain_read_count; void *chain_info; }; + struct + { + // for scrub: parts left when no part of the target version is found + obj_ver_osd_t *scrub_parts; + int scrub_part_count; + }; }; }; diff --git a/src/osd/osd_primary_write.cpp b/src/osd/osd_primary_write.cpp index 1147bee3..0a70bf01 100644 --- a/src/osd/osd_primary_write.cpp +++ b/src/osd/osd_primary_write.cpp @@ -264,7 +264,11 @@ resume_3: std::sort(history_set.begin(), history_set.end()); auto it = std::lower_bound(pg.target_history.begin(), pg.target_history.end(), history_set); if (it == pg.target_history.end() || *it != history_set) + { pg.target_history.insert(it, history_set); + // Like in peering: otherwise the PG reconciling loop sees the old history and stops the PG + st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history; + } } pg.history_changed = true; report_pg_states(); diff --git a/src/osd/osd_scrub.cpp b/src/osd/osd_scrub.cpp index 112813c7..50e008c1 100644 --- a/src/osd/osd_scrub.cpp +++ b/src/osd/osd_scrub.cpp @@ -446,6 +446,43 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) } if (!total) { + pg_osd_set_state_t *object_state = NULL; + get_object_osd_set(*op_data->pg, op_data->oid, &object_state); + if (object_state != op_data->object_state) + { + // Object state changed in between by a parallel I/O operation + return; + } + int left = 0; + if (object_state) + { + for (auto & chunk: object_state->osd_set) + { + if (chunk.loc_bad & LOC_OUTDATED) + left++; + } + } + if (left > 0) + { + // All parts of the target version are deleted manually, but older ones are left: + // continue_primary_scrub() learns their versions and recalculates the object state + op_data->scrub_parts = (obj_ver_osd_t*)calloc_or_die(left, sizeof(obj_ver_osd_t)); + op_data->scrub_part_count = 0; + for (auto & chunk: object_state->osd_set) + { + if (chunk.loc_bad & LOC_OUTDATED) + { + op_data->scrub_parts[op_data->scrub_part_count++] = (obj_ver_osd_t){ + .osd_num = chunk.osd_num, + .oid = { + .inode = op_data->oid.inode, + .stripe = op_data->oid.stripe | (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role), + }, + }; + } + } + return; + } // Object is deleted manually from all OSDs, forget it printf( "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n", @@ -453,7 +490,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) op_data->oid.inode, op_data->oid.stripe ); remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false); - deref_object_state(*op_data->pg, &op_data->object_state, true); + deref_object_state(*op_data->pg, &op_data->object_state, false); return; } if (op_data->pg->scheme == POOL_SCHEME_REPLICATED) @@ -650,6 +687,121 @@ void osd_t::scrub_check_results(osd_op_t *cur_op) }); } +// Ask OSDs for versions of the parts left in scrub_parts. Returns false if all of them are local +bool osd_t::submit_scrub_version_subops(osd_op_t *cur_op) +{ + osd_primary_op_data_t *op_data = cur_op->op_data; + int remote = 0; + for (int i = 0; i < op_data->scrub_part_count; i++) + { + obj_ver_osd_t & part = op_data->scrub_parts[i]; + if (part.osd_num == this->osd_num) + bs->read_bitmap(part.oid, UINT64_MAX, NULL, &part.version); + else + remote++; + } + op_data->done = op_data->errors = op_data->errcode = 0; + op_data->n_subops = remote; + op_data->st = 3; + if (!remote) + { + return false; + } + op_data->subops = new osd_op_t[remote]; + // The last subop may complete the operation, so op_data is not touched after it's submitted + for (int i = 0, subop_idx = 0; subop_idx < remote; i++) + { + obj_ver_osd_t *part = &op_data->scrub_parts[i]; + if (part->osd_num == this->osd_num) + continue; + osd_op_t *subop = &op_data->subops[subop_idx++]; + subop->op_type = OSD_OP_OUT; + subop->buf = malloc_or_die(sizeof(obj_ver_id)); + *(obj_ver_id*)subop->buf = (obj_ver_id){ .oid = part->oid, .version = UINT64_MAX }; + subop->req = (osd_any_op_t){ + .sec_read_bmp = { + .header = { + .magic = SECONDARY_OSD_OP_MAGIC, + .opcode = OSD_OP_SEC_READ_BMP, + }, + .len = sizeof(obj_ver_id), + } + }; + subop->callback = [this, cur_op, part](osd_op_t *subop) + { + if (subop->reply.hdr.retval == 8 + clean_entry_bitmap_size) + memcpy(&part->version, subop->buf, 8); + handle_primary_subop(subop, cur_op); + }; + submit_to_osd(subop, part->osd_num); + } + return true; +} + +// Recalculate the object state from the parts left: the newest version becomes the target, +// like in peering, older ones stay outdated, and parts which are gone are removed +void osd_t::scrub_recalc_state(osd_op_t *cur_op) +{ + osd_primary_op_data_t *op_data = cur_op->op_data; + uint64_t target_ver = 0; + for (int i = 0; i < op_data->scrub_part_count; i++) + { + if (op_data->scrub_parts[i].version > target_ver) + target_ver = op_data->scrub_parts[i].version; + } + if (!target_ver) + { + printf( + "[PG %u/%u] Scrub detected a deleted object %jx:%jx\n", + INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe + ); + pg_osd_set_state_t *object_state = NULL; + get_object_osd_set(*op_data->pg, op_data->oid, &object_state); + if (object_state == op_data->object_state) + { + remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false); + deref_object_state(*op_data->pg, &op_data->object_state, false); + } + return; + } + printf( + "[PG %u/%u] Scrub detected deleted parts of object %jx:%jx, recalculating its state from version %ju\n", + INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe, target_ver + ); + op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false, [op_data, target_ver](pg_osd_set_t & new_set) + { + int changes = 0; + for (int i = new_set.size()-1; i >= 0; i--) + { + uint64_t version = 0; + if (new_set[i].loc_bad & LOC_OUTDATED) + { + for (int j = 0; j < op_data->scrub_part_count; j++) + { + if (op_data->scrub_parts[j].osd_num == new_set[i].osd_num && + (op_data->scrub_parts[j].oid.stripe & STRIPE_MASK) == (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : new_set[i].role)) + { + version = op_data->scrub_parts[j].version; + break; + } + } + } + if (!version) + { + // Target version parts are checked by scrub as deleted, other parts are gone + new_set.erase(new_set.begin()+i, new_set.begin()+i+1); + changes++; + } + else if (version == target_ver) + { + new_set[i].loc_bad &= ~LOC_OUTDATED; + changes++; + } + } + return changes; + }); +} + void osd_t::continue_primary_scrub(osd_op_t *cur_op) { if (!cur_op->op_data && !prepare_primary_rw(cur_op)) @@ -658,6 +810,10 @@ void osd_t::continue_primary_scrub(osd_op_t *cur_op) goto resume_1; else if (cur_op->op_data->st == 2) goto resume_2; + else if (cur_op->op_data->st == 3) + goto resume_1; + else if (cur_op->op_data->st == 4) + goto resume_4; submit_scrub_subops(cur_op); resume_1: return; @@ -677,5 +833,17 @@ resume_2: free(cur_op->op_data->stripes); cur_op->op_data->stripes = NULL; } + if (cur_op->op_data->scrub_parts && submit_scrub_version_subops(cur_op)) + { + return; + } +resume_4: + if (cur_op->op_data->scrub_parts) + { + if (!cur_op->op_data->errors) + scrub_recalc_state(cur_op); + free(cur_op->op_data->scrub_parts); + cur_op->op_data->scrub_parts = NULL; + } finish_op(cur_op, cur_op->op_data->errcode); } diff --git a/tests/run_tests.sh b/tests/run_tests.sh index f77fb33d..dc4c6577 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -167,6 +167,7 @@ SCHEME=xor ./test_scrub.sh PG_SIZE=3 ./test_scrub.sh PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh SCHEME=ec ./test_scrub.sh +./test_scrub_deleted_incomplete.sh OLD=1 ./test_scrub.sh OLD=1 ZERO_OSD=2 ./test_scrub.sh diff --git a/tests/test_scrub_deleted_incomplete.sh b/tests/test_scrub_deleted_incomplete.sh new file mode 100755 index 00000000..4703de8c --- /dev/null +++ b/tests/test_scrub_deleted_incomplete.sh @@ -0,0 +1,69 @@ +#!/bin/bash -ex + +# Deleting the only chunk of an incomplete EC object's newest version with fix --check no +# must not make scrub forget the object: its previous version is still on the other OSDs, +# and fix must make the object state correct online, without re-peering the PG + +SCHEME=ec +IMMEDIATE_COMMIT=1 +. `dirname $0`/run_3osds.sh + +OBJ_SIZE=$((128*1024*PG_DATA_SIZE)) +LONE=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[0]') +OTHERS=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[1:][]') + +stop_others() +{ + for i in $OTHERS; do + eval kill -9 \$OSD${i}_PID + eval wait \$OSD${i}_PID || true + $ETCDCTL del /vitastor/osd/state/$i + done +} + +$VITASTOR_CLI create -s $OBJ_SIZE testimg +dd if=/dev/urandom of=./testdata/bin/a.bin bs=$OBJ_SIZE count=1 +dd if=/dev/urandom of=./testdata/bin/b.bin bs=$OBJ_SIZE count=1 + +# Version A on all OSDs, saved on all OSDs except $LONE +$VITASTOR_CLI dd if=./testdata/bin/a.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat +stop_others +for i in $OTHERS; do + cp --sparse=always ./testdata/bin/test_osd$i.bin ./testdata/bin/saved_osd$i.bin +done +for i in $OTHERS; do + start_osd $i +done +wait_up 60 + +# Version B on all OSDs, then roll the other OSDs back to version A: +# B is left only as part 0 on $LONE and the object becomes incomplete +$VITASTOR_CLI dd if=./testdata/bin/b.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat +stop_others +for i in $OTHERS; do + cp --sparse=always ./testdata/bin/saved_osd$i.bin ./testdata/bin/test_osd$i.bin + start_osd $i +done +wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -e '(.state | index(\"active\")) and (.state | index(\"has_incomplete\"))'" "PG peering with rolled back OSDs" +OBJ=$($VITASTOR_CLI describe --object-state incomplete --json | jq -r -e '.[0] | .inode+":"+.stripe') + +# Removing B leaves 4 parts of A, enough to read it +PRIMARY=$($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -r .primary) +PEERINGS=$(grep -c 'is peering' ./testdata/osd$PRIMARY.log) +$VITASTOR_CLI fix --objects $OBJ --bad-osds $LONE --part 0 --check no +$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE +if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then + format_error "Object read after fix differs from version A" +fi + +# Recovery must restore part 0 of A on $LONE +wait_up 60 +$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE +if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then + format_error "Object read after recovery differs from version A" +fi +if [[ $(grep -c 'is peering' ./testdata/osd$PRIMARY.log) != $PEERINGS ]]; then + format_error "PG was re-peered after fix" +fi + +format_green OK