osd: Recalculate object state online when scrub finds its target version deleted #118

Merged
vitalif merged 2 commits from koolkhel/vitastor:scrub-deleted-repeer into master 2026-11-17 10:20:48 +03:00
8 changed files with 272 additions and 3 deletions

View File

@@ -2183,6 +2183,24 @@ jobs:
echo ""
done
test_scrub_deleted_incomplete:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_scrub_deleted_incomplete.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_old_scrub:
runs-on: ubuntu-latest
needs: build

View File

@@ -344,6 +344,8 @@ class osd_t
bool continue_scrub();
void submit_scrub_subops(osd_op_t *cur_op);
void scrub_check_results(osd_op_t *cur_op);
bool submit_scrub_version_subops(osd_op_t *cur_op);
void scrub_recalc_state(osd_op_t *cur_op);
void plan_scrub(pg_t & pg, bool report_state = true);
void schedule_scrub(pg_t & pg);

View File

@@ -291,7 +291,8 @@ void pg_obj_state_check_t::finish_object()
}
if (ver_end < obj_end)
{
// Check for outdated versions not present in the current target OSD set
// The state lists every OSD with any version of the object, each role on an OSD once.
// Older pieces on their target OSDs are listed too: if the target version is lost, they're all that is left
for (int i = ver_end; i < obj_end; i++)
{
int j;
@@ -307,7 +308,7 @@ void pg_obj_state_check_t::finish_object()
break;
}
}
if (j >= osd_set.size() && pg->cur_set[list[i].oid.stripe & STRIPE_MASK] != list[i].osd_num)
if (j >= osd_set.size())
{
osd_set.push_back((pg_obj_loc_t){
.role = (list[i].oid.stripe & STRIPE_MASK),

View File

@@ -62,6 +62,12 @@ struct osd_primary_op_data_t
int chain_read_count;
void *chain_info;
};
struct
{
// for scrub: parts left when no part of the target version is found
obj_ver_osd_t *scrub_parts;
int scrub_part_count;
};
};
};

View File

@@ -264,7 +264,11 @@ resume_3:
std::sort(history_set.begin(), history_set.end());
auto it = std::lower_bound(pg.target_history.begin(), pg.target_history.end(), history_set);
if (it == pg.target_history.end() || *it != history_set)
{
pg.target_history.insert(it, history_set);
// Like in peering: otherwise the PG reconciling loop sees the old history and stops the PG
st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history;
}
}
pg.history_changed = true;
report_pg_states();

View File

@@ -446,6 +446,43 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
}
if (!total)
{
pg_osd_set_state_t *object_state = NULL;
get_object_osd_set(*op_data->pg, op_data->oid, &object_state);
if (object_state != op_data->object_state)
{
// Object state changed in between by a parallel I/O operation
return;
}
int left = 0;
if (object_state)
{
for (auto & chunk: object_state->osd_set)
{
if (chunk.loc_bad & LOC_OUTDATED)
left++;
}
}
if (left > 0)
{
// All parts of the target version are deleted manually, but older ones are left:
// continue_primary_scrub() learns their versions and recalculates the object state
op_data->scrub_parts = (obj_ver_osd_t*)calloc_or_die(left, sizeof(obj_ver_osd_t));
op_data->scrub_part_count = 0;
for (auto & chunk: object_state->osd_set)
{
if (chunk.loc_bad & LOC_OUTDATED)
{
op_data->scrub_parts[op_data->scrub_part_count++] = (obj_ver_osd_t){
.osd_num = chunk.osd_num,
.oid = {
.inode = op_data->oid.inode,
.stripe = op_data->oid.stripe | (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
},
};
}
}
return;
}
// Object is deleted manually from all OSDs, forget it
printf(
"[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
@@ -453,7 +490,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
op_data->oid.inode, op_data->oid.stripe
);
remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
deref_object_state(*op_data->pg, &op_data->object_state, true);
deref_object_state(*op_data->pg, &op_data->object_state, false);
return;
}
if (op_data->pg->scheme == POOL_SCHEME_REPLICATED)
@@ -650,6 +687,121 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
});
}
// Ask OSDs for versions of the parts left in scrub_parts. Returns false if all of them are local
bool osd_t::submit_scrub_version_subops(osd_op_t *cur_op)
{
osd_primary_op_data_t *op_data = cur_op->op_data;
int remote = 0;
for (int i = 0; i < op_data->scrub_part_count; i++)
{
obj_ver_osd_t & part = op_data->scrub_parts[i];
if (part.osd_num == this->osd_num)
bs->read_bitmap(part.oid, UINT64_MAX, NULL, &part.version);
else
remote++;
}
op_data->done = op_data->errors = op_data->errcode = 0;
op_data->n_subops = remote;
op_data->st = 3;
if (!remote)
{
return false;
}
op_data->subops = new osd_op_t[remote];
// The last subop may complete the operation, so op_data is not touched after it's submitted
for (int i = 0, subop_idx = 0; subop_idx < remote; i++)
{
obj_ver_osd_t *part = &op_data->scrub_parts[i];
if (part->osd_num == this->osd_num)
continue;
osd_op_t *subop = &op_data->subops[subop_idx++];
subop->op_type = OSD_OP_OUT;
subop->buf = malloc_or_die(sizeof(obj_ver_id));
*(obj_ver_id*)subop->buf = (obj_ver_id){ .oid = part->oid, .version = UINT64_MAX };
subop->req = (osd_any_op_t){
.sec_read_bmp = {
.header = {
.magic = SECONDARY_OSD_OP_MAGIC,
.opcode = OSD_OP_SEC_READ_BMP,
},
.len = sizeof(obj_ver_id),
}
};
subop->callback = [this, cur_op, part](osd_op_t *subop)
{
if (subop->reply.hdr.retval == 8 + clean_entry_bitmap_size)
memcpy(&part->version, subop->buf, 8);
handle_primary_subop(subop, cur_op);
};
submit_to_osd(subop, part->osd_num);
}
return true;
}
// Recalculate the object state from the parts left: the newest version becomes the target,
// like in peering, older ones stay outdated, and parts which are gone are removed
void osd_t::scrub_recalc_state(osd_op_t *cur_op)
{
osd_primary_op_data_t *op_data = cur_op->op_data;
uint64_t target_ver = 0;
for (int i = 0; i < op_data->scrub_part_count; i++)
{
if (op_data->scrub_parts[i].version > target_ver)
target_ver = op_data->scrub_parts[i].version;
}
if (!target_ver)
{
printf(
"[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe
);
pg_osd_set_state_t *object_state = NULL;
get_object_osd_set(*op_data->pg, op_data->oid, &object_state);
if (object_state == op_data->object_state)
{
remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
deref_object_state(*op_data->pg, &op_data->object_state, false);
}
return;
}
printf(
"[PG %u/%u] Scrub detected deleted parts of object %jx:%jx, recalculating its state from version %ju\n",
INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe, target_ver
);
op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false, [op_data, target_ver](pg_osd_set_t & new_set)
{
int changes = 0;
for (int i = new_set.size()-1; i >= 0; i--)
{
uint64_t version = 0;
if (new_set[i].loc_bad & LOC_OUTDATED)
{
for (int j = 0; j < op_data->scrub_part_count; j++)
{
if (op_data->scrub_parts[j].osd_num == new_set[i].osd_num &&
(op_data->scrub_parts[j].oid.stripe & STRIPE_MASK) == (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : new_set[i].role))
{
version = op_data->scrub_parts[j].version;
break;
}
}
}
if (!version)
{
// Target version parts are checked by scrub as deleted, other parts are gone
new_set.erase(new_set.begin()+i, new_set.begin()+i+1);
changes++;
}
else if (version == target_ver)
{
new_set[i].loc_bad &= ~LOC_OUTDATED;
changes++;
}
}
return changes;
});
}
void osd_t::continue_primary_scrub(osd_op_t *cur_op)
{
if (!cur_op->op_data && !prepare_primary_rw(cur_op))
@@ -658,6 +810,10 @@ void osd_t::continue_primary_scrub(osd_op_t *cur_op)
goto resume_1;
else if (cur_op->op_data->st == 2)
goto resume_2;
else if (cur_op->op_data->st == 3)
goto resume_1;
else if (cur_op->op_data->st == 4)
goto resume_4;
submit_scrub_subops(cur_op);
resume_1:
return;
@@ -677,5 +833,17 @@ resume_2:
free(cur_op->op_data->stripes);
cur_op->op_data->stripes = NULL;
}
if (cur_op->op_data->scrub_parts && submit_scrub_version_subops(cur_op))
{
return;
}
resume_4:
if (cur_op->op_data->scrub_parts)
{
if (!cur_op->op_data->errors)
scrub_recalc_state(cur_op);
free(cur_op->op_data->scrub_parts);
cur_op->op_data->scrub_parts = NULL;
}
finish_op(cur_op, cur_op->op_data->errcode);
}

View File

@@ -167,6 +167,7 @@ SCHEME=xor ./test_scrub.sh
PG_SIZE=3 ./test_scrub.sh
PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh
SCHEME=ec ./test_scrub.sh
./test_scrub_deleted_incomplete.sh
OLD=1 ./test_scrub.sh
OLD=1 ZERO_OSD=2 ./test_scrub.sh

View File

@@ -0,0 +1,69 @@
#!/bin/bash -ex
# Deleting the only chunk of an incomplete EC object's newest version with fix --check no
# must not make scrub forget the object: its previous version is still on the other OSDs,
# and fix must make the object state correct online, without re-peering the PG
SCHEME=ec
IMMEDIATE_COMMIT=1
. `dirname $0`/run_3osds.sh
OBJ_SIZE=$((128*1024*PG_DATA_SIZE))
LONE=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[0]')
OTHERS=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[1:][]')
stop_others()
{
for i in $OTHERS; do
eval kill -9 \$OSD${i}_PID
eval wait \$OSD${i}_PID || true
$ETCDCTL del /vitastor/osd/state/$i
done
}
$VITASTOR_CLI create -s $OBJ_SIZE testimg
dd if=/dev/urandom of=./testdata/bin/a.bin bs=$OBJ_SIZE count=1
dd if=/dev/urandom of=./testdata/bin/b.bin bs=$OBJ_SIZE count=1
# Version A on all OSDs, saved on all OSDs except $LONE
$VITASTOR_CLI dd if=./testdata/bin/a.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat
stop_others
for i in $OTHERS; do
cp --sparse=always ./testdata/bin/test_osd$i.bin ./testdata/bin/saved_osd$i.bin
done
for i in $OTHERS; do
start_osd $i
done
wait_up 60
# Version B on all OSDs, then roll the other OSDs back to version A:
# B is left only as part 0 on $LONE and the object becomes incomplete
$VITASTOR_CLI dd if=./testdata/bin/b.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat
stop_others
for i in $OTHERS; do
cp --sparse=always ./testdata/bin/saved_osd$i.bin ./testdata/bin/test_osd$i.bin
start_osd $i
done
wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -e '(.state | index(\"active\")) and (.state | index(\"has_incomplete\"))'" "PG peering with rolled back OSDs"
OBJ=$($VITASTOR_CLI describe --object-state incomplete --json | jq -r -e '.[0] | .inode+":"+.stripe')
# Removing B leaves 4 parts of A, enough to read it
PRIMARY=$($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -r .primary)
PEERINGS=$(grep -c 'is peering' ./testdata/osd$PRIMARY.log)
$VITASTOR_CLI fix --objects $OBJ --bad-osds $LONE --part 0 --check no
$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE
if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then
format_error "Object read after fix differs from version A"
fi
# Recovery must restore part 0 of A on $LONE
wait_up 60
$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE
if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then
format_error "Object read after recovery differs from version A"
fi
if [[ $(grep -c 'is peering' ./testdata/osd$PRIMARY.log) != $PEERINGS ]]; then
format_error "PG was re-peered after fix"
fi
format_green OK