osd: Recalculate object state online when scrub finds its target version deleted #118
@@ -2183,6 +2183,24 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_scrub_deleted_incomplete:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_scrub_deleted_incomplete.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_old_scrub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
||||
@@ -344,6 +344,8 @@ class osd_t
|
||||
bool continue_scrub();
|
||||
void submit_scrub_subops(osd_op_t *cur_op);
|
||||
void scrub_check_results(osd_op_t *cur_op);
|
||||
bool submit_scrub_version_subops(osd_op_t *cur_op);
|
||||
void scrub_recalc_state(osd_op_t *cur_op);
|
||||
void plan_scrub(pg_t & pg, bool report_state = true);
|
||||
void schedule_scrub(pg_t & pg);
|
||||
|
||||
|
||||
@@ -291,7 +291,8 @@ void pg_obj_state_check_t::finish_object()
|
||||
}
|
||||
if (ver_end < obj_end)
|
||||
{
|
||||
// Check for outdated versions not present in the current target OSD set
|
||||
// The state lists every OSD with any version of the object, each role on an OSD once.
|
||||
// Older pieces on their target OSDs are listed too: if the target version is lost, they're all that is left
|
||||
for (int i = ver_end; i < obj_end; i++)
|
||||
{
|
||||
int j;
|
||||
@@ -307,7 +308,7 @@ void pg_obj_state_check_t::finish_object()
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (j >= osd_set.size() && pg->cur_set[list[i].oid.stripe & STRIPE_MASK] != list[i].osd_num)
|
||||
if (j >= osd_set.size())
|
||||
{
|
||||
osd_set.push_back((pg_obj_loc_t){
|
||||
.role = (list[i].oid.stripe & STRIPE_MASK),
|
||||
|
||||
@@ -62,6 +62,12 @@ struct osd_primary_op_data_t
|
||||
int chain_read_count;
|
||||
void *chain_info;
|
||||
};
|
||||
struct
|
||||
{
|
||||
// for scrub: parts left when no part of the target version is found
|
||||
obj_ver_osd_t *scrub_parts;
|
||||
int scrub_part_count;
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
|
||||
@@ -264,7 +264,11 @@ resume_3:
|
||||
std::sort(history_set.begin(), history_set.end());
|
||||
auto it = std::lower_bound(pg.target_history.begin(), pg.target_history.end(), history_set);
|
||||
if (it == pg.target_history.end() || *it != history_set)
|
||||
{
|
||||
pg.target_history.insert(it, history_set);
|
||||
// Like in peering: otherwise the PG reconciling loop sees the old history and stops the PG
|
||||
st_cli->pool_config[pg.pool_id].pg_config[pg.pg_num].target_history = pg.target_history;
|
||||
}
|
||||
}
|
||||
pg.history_changed = true;
|
||||
report_pg_states();
|
||||
|
||||
@@ -446,6 +446,43 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
|
||||
}
|
||||
if (!total)
|
||||
{
|
||||
pg_osd_set_state_t *object_state = NULL;
|
||||
get_object_osd_set(*op_data->pg, op_data->oid, &object_state);
|
||||
if (object_state != op_data->object_state)
|
||||
{
|
||||
// Object state changed in between by a parallel I/O operation
|
||||
return;
|
||||
}
|
||||
int left = 0;
|
||||
if (object_state)
|
||||
{
|
||||
for (auto & chunk: object_state->osd_set)
|
||||
{
|
||||
if (chunk.loc_bad & LOC_OUTDATED)
|
||||
left++;
|
||||
}
|
||||
}
|
||||
if (left > 0)
|
||||
{
|
||||
// All parts of the target version are deleted manually, but older ones are left:
|
||||
// continue_primary_scrub() learns their versions and recalculates the object state
|
||||
op_data->scrub_parts = (obj_ver_osd_t*)calloc_or_die(left, sizeof(obj_ver_osd_t));
|
||||
op_data->scrub_part_count = 0;
|
||||
for (auto & chunk: object_state->osd_set)
|
||||
{
|
||||
if (chunk.loc_bad & LOC_OUTDATED)
|
||||
{
|
||||
op_data->scrub_parts[op_data->scrub_part_count++] = (obj_ver_osd_t){
|
||||
.osd_num = chunk.osd_num,
|
||||
.oid = {
|
||||
.inode = op_data->oid.inode,
|
||||
.stripe = op_data->oid.stripe | (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : chunk.role),
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Object is deleted manually from all OSDs, forget it
|
||||
printf(
|
||||
"[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
|
||||
@@ -453,7 +490,7 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
|
||||
op_data->oid.inode, op_data->oid.stripe
|
||||
);
|
||||
remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
|
||||
deref_object_state(*op_data->pg, &op_data->object_state, true);
|
||||
deref_object_state(*op_data->pg, &op_data->object_state, false);
|
||||
return;
|
||||
}
|
||||
if (op_data->pg->scheme == POOL_SCHEME_REPLICATED)
|
||||
@@ -650,6 +687,121 @@ void osd_t::scrub_check_results(osd_op_t *cur_op)
|
||||
});
|
||||
}
|
||||
|
||||
// Ask OSDs for versions of the parts left in scrub_parts. Returns false if all of them are local
|
||||
bool osd_t::submit_scrub_version_subops(osd_op_t *cur_op)
|
||||
{
|
||||
osd_primary_op_data_t *op_data = cur_op->op_data;
|
||||
int remote = 0;
|
||||
for (int i = 0; i < op_data->scrub_part_count; i++)
|
||||
{
|
||||
obj_ver_osd_t & part = op_data->scrub_parts[i];
|
||||
if (part.osd_num == this->osd_num)
|
||||
bs->read_bitmap(part.oid, UINT64_MAX, NULL, &part.version);
|
||||
else
|
||||
remote++;
|
||||
}
|
||||
op_data->done = op_data->errors = op_data->errcode = 0;
|
||||
op_data->n_subops = remote;
|
||||
op_data->st = 3;
|
||||
if (!remote)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
op_data->subops = new osd_op_t[remote];
|
||||
// The last subop may complete the operation, so op_data is not touched after it's submitted
|
||||
for (int i = 0, subop_idx = 0; subop_idx < remote; i++)
|
||||
{
|
||||
obj_ver_osd_t *part = &op_data->scrub_parts[i];
|
||||
if (part->osd_num == this->osd_num)
|
||||
continue;
|
||||
osd_op_t *subop = &op_data->subops[subop_idx++];
|
||||
subop->op_type = OSD_OP_OUT;
|
||||
subop->buf = malloc_or_die(sizeof(obj_ver_id));
|
||||
*(obj_ver_id*)subop->buf = (obj_ver_id){ .oid = part->oid, .version = UINT64_MAX };
|
||||
subop->req = (osd_any_op_t){
|
||||
.sec_read_bmp = {
|
||||
.header = {
|
||||
.magic = SECONDARY_OSD_OP_MAGIC,
|
||||
.opcode = OSD_OP_SEC_READ_BMP,
|
||||
},
|
||||
.len = sizeof(obj_ver_id),
|
||||
}
|
||||
};
|
||||
subop->callback = [this, cur_op, part](osd_op_t *subop)
|
||||
{
|
||||
if (subop->reply.hdr.retval == 8 + clean_entry_bitmap_size)
|
||||
memcpy(&part->version, subop->buf, 8);
|
||||
handle_primary_subop(subop, cur_op);
|
||||
};
|
||||
submit_to_osd(subop, part->osd_num);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Recalculate the object state from the parts left: the newest version becomes the target,
|
||||
// like in peering, older ones stay outdated, and parts which are gone are removed
|
||||
void osd_t::scrub_recalc_state(osd_op_t *cur_op)
|
||||
{
|
||||
osd_primary_op_data_t *op_data = cur_op->op_data;
|
||||
uint64_t target_ver = 0;
|
||||
for (int i = 0; i < op_data->scrub_part_count; i++)
|
||||
{
|
||||
if (op_data->scrub_parts[i].version > target_ver)
|
||||
target_ver = op_data->scrub_parts[i].version;
|
||||
}
|
||||
if (!target_ver)
|
||||
{
|
||||
printf(
|
||||
"[PG %u/%u] Scrub detected a deleted object %jx:%jx\n",
|
||||
INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe
|
||||
);
|
||||
pg_osd_set_state_t *object_state = NULL;
|
||||
get_object_osd_set(*op_data->pg, op_data->oid, &object_state);
|
||||
if (object_state == op_data->object_state)
|
||||
{
|
||||
remove_object_from_state(op_data->oid, &op_data->object_state, *op_data->pg, false);
|
||||
deref_object_state(*op_data->pg, &op_data->object_state, false);
|
||||
}
|
||||
return;
|
||||
}
|
||||
printf(
|
||||
"[PG %u/%u] Scrub detected deleted parts of object %jx:%jx, recalculating its state from version %ju\n",
|
||||
INODE_POOL(op_data->oid.inode), op_data->pg_num, op_data->oid.inode, op_data->oid.stripe, target_ver
|
||||
);
|
||||
op_data->object_state = mark_object(*op_data->pg, op_data->oid, op_data->object_state, false, [op_data, target_ver](pg_osd_set_t & new_set)
|
||||
{
|
||||
int changes = 0;
|
||||
for (int i = new_set.size()-1; i >= 0; i--)
|
||||
{
|
||||
uint64_t version = 0;
|
||||
if (new_set[i].loc_bad & LOC_OUTDATED)
|
||||
{
|
||||
for (int j = 0; j < op_data->scrub_part_count; j++)
|
||||
{
|
||||
if (op_data->scrub_parts[j].osd_num == new_set[i].osd_num &&
|
||||
(op_data->scrub_parts[j].oid.stripe & STRIPE_MASK) == (op_data->pg->scheme == POOL_SCHEME_REPLICATED ? 0 : new_set[i].role))
|
||||
{
|
||||
version = op_data->scrub_parts[j].version;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (!version)
|
||||
{
|
||||
// Target version parts are checked by scrub as deleted, other parts are gone
|
||||
new_set.erase(new_set.begin()+i, new_set.begin()+i+1);
|
||||
changes++;
|
||||
}
|
||||
else if (version == target_ver)
|
||||
{
|
||||
new_set[i].loc_bad &= ~LOC_OUTDATED;
|
||||
changes++;
|
||||
}
|
||||
}
|
||||
return changes;
|
||||
});
|
||||
}
|
||||
|
||||
void osd_t::continue_primary_scrub(osd_op_t *cur_op)
|
||||
{
|
||||
if (!cur_op->op_data && !prepare_primary_rw(cur_op))
|
||||
@@ -658,6 +810,10 @@ void osd_t::continue_primary_scrub(osd_op_t *cur_op)
|
||||
goto resume_1;
|
||||
else if (cur_op->op_data->st == 2)
|
||||
goto resume_2;
|
||||
else if (cur_op->op_data->st == 3)
|
||||
goto resume_1;
|
||||
else if (cur_op->op_data->st == 4)
|
||||
goto resume_4;
|
||||
submit_scrub_subops(cur_op);
|
||||
resume_1:
|
||||
return;
|
||||
@@ -677,5 +833,17 @@ resume_2:
|
||||
free(cur_op->op_data->stripes);
|
||||
cur_op->op_data->stripes = NULL;
|
||||
}
|
||||
if (cur_op->op_data->scrub_parts && submit_scrub_version_subops(cur_op))
|
||||
{
|
||||
return;
|
||||
}
|
||||
resume_4:
|
||||
if (cur_op->op_data->scrub_parts)
|
||||
{
|
||||
if (!cur_op->op_data->errors)
|
||||
scrub_recalc_state(cur_op);
|
||||
free(cur_op->op_data->scrub_parts);
|
||||
cur_op->op_data->scrub_parts = NULL;
|
||||
}
|
||||
finish_op(cur_op, cur_op->op_data->errcode);
|
||||
}
|
||||
|
||||
@@ -167,6 +167,7 @@ SCHEME=xor ./test_scrub.sh
|
||||
PG_SIZE=3 ./test_scrub.sh
|
||||
PG_SIZE=6 PG_MINSIZE=4 OSD_COUNT=6 SCHEME=ec ./test_scrub.sh
|
||||
SCHEME=ec ./test_scrub.sh
|
||||
./test_scrub_deleted_incomplete.sh
|
||||
|
||||
OLD=1 ./test_scrub.sh
|
||||
OLD=1 ZERO_OSD=2 ./test_scrub.sh
|
||||
|
||||
69
tests/test_scrub_deleted_incomplete.sh
Executable file
69
tests/test_scrub_deleted_incomplete.sh
Executable file
@@ -0,0 +1,69 @@
|
||||
#!/bin/bash -ex
|
||||
|
||||
# Deleting the only chunk of an incomplete EC object's newest version with fix --check no
|
||||
# must not make scrub forget the object: its previous version is still on the other OSDs,
|
||||
# and fix must make the object state correct online, without re-peering the PG
|
||||
|
||||
SCHEME=ec
|
||||
IMMEDIATE_COMMIT=1
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
OBJ_SIZE=$((128*1024*PG_DATA_SIZE))
|
||||
LONE=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[0]')
|
||||
OTHERS=$($ETCDCTL get /vitastor/pg/config --print-value-only | jq -r '.items["1"]["1"].osd_set[1:][]')
|
||||
|
||||
stop_others()
|
||||
{
|
||||
for i in $OTHERS; do
|
||||
eval kill -9 \$OSD${i}_PID
|
||||
eval wait \$OSD${i}_PID || true
|
||||
$ETCDCTL del /vitastor/osd/state/$i
|
||||
done
|
||||
}
|
||||
|
||||
$VITASTOR_CLI create -s $OBJ_SIZE testimg
|
||||
dd if=/dev/urandom of=./testdata/bin/a.bin bs=$OBJ_SIZE count=1
|
||||
dd if=/dev/urandom of=./testdata/bin/b.bin bs=$OBJ_SIZE count=1
|
||||
|
||||
# Version A on all OSDs, saved on all OSDs except $LONE
|
||||
$VITASTOR_CLI dd if=./testdata/bin/a.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat
|
||||
stop_others
|
||||
for i in $OTHERS; do
|
||||
cp --sparse=always ./testdata/bin/test_osd$i.bin ./testdata/bin/saved_osd$i.bin
|
||||
done
|
||||
for i in $OTHERS; do
|
||||
start_osd $i
|
||||
done
|
||||
wait_up 60
|
||||
|
||||
# Version B on all OSDs, then roll the other OSDs back to version A:
|
||||
# B is left only as part 0 on $LONE and the object becomes incomplete
|
||||
$VITASTOR_CLI dd if=./testdata/bin/b.bin oimg=testimg bs=$OBJ_SIZE conv=nocreat
|
||||
stop_others
|
||||
for i in $OTHERS; do
|
||||
cp --sparse=always ./testdata/bin/saved_osd$i.bin ./testdata/bin/test_osd$i.bin
|
||||
start_osd $i
|
||||
done
|
||||
wait_condition 60 "$ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -e '(.state | index(\"active\")) and (.state | index(\"has_incomplete\"))'" "PG peering with rolled back OSDs"
|
||||
OBJ=$($VITASTOR_CLI describe --object-state incomplete --json | jq -r -e '.[0] | .inode+":"+.stripe')
|
||||
|
||||
# Removing B leaves 4 parts of A, enough to read it
|
||||
PRIMARY=$($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -r .primary)
|
||||
PEERINGS=$(grep -c 'is peering' ./testdata/osd$PRIMARY.log)
|
||||
$VITASTOR_CLI fix --objects $OBJ --bad-osds $LONE --part 0 --check no
|
||||
$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE
|
||||
if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then
|
||||
format_error "Object read after fix differs from version A"
|
||||
fi
|
||||
|
||||
# Recovery must restore part 0 of A on $LONE
|
||||
wait_up 60
|
||||
$VITASTOR_CLI dd iimg=testimg of=./testdata/bin/read.bin bs=$OBJ_SIZE
|
||||
if ! cmp ./testdata/bin/a.bin ./testdata/bin/read.bin; then
|
||||
format_error "Object read after recovery differs from version A"
|
||||
fi
|
||||
if [[ $(grep -c 'is peering' ./testdata/osd$PRIMARY.log) != $PEERINGS ]]; then
|
||||
format_error "PG was re-peered after fix"
|
||||
fi
|
||||
|
||||
format_green OK
|
||||
Reference in New Issue
Block a user