osd: Pause recovery of a PG after an error instead of retrying it in a loop #143
@@ -2075,6 +2075,24 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_recovery_enospc:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_recovery_enospc.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_scrub:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
||||
@@ -107,6 +107,11 @@ osd_t::~osd_t()
|
||||
tfd->clear_timer(autosync_timer_id);
|
||||
autosync_timer_id = -1;
|
||||
}
|
||||
if (recovery_retry_timer_id >= 0)
|
||||
{
|
||||
tfd->clear_timer(recovery_retry_timer_id);
|
||||
recovery_retry_timer_id = -1;
|
||||
}
|
||||
ringloop->unregister_consumer(&consumer);
|
||||
ringloop->unregister_consumer(&init_consumer);
|
||||
// Tear down clients first: destroying a client cancels its operations and the
|
||||
|
||||
@@ -38,6 +38,9 @@
|
||||
#define DEFAULT_RECOVERY_QUEUE 1
|
||||
#define DEFAULT_RECOVERY_PG_SWITCH 128
|
||||
#define DEFAULT_RECOVERY_BATCH 16
|
||||
// A PG's recovery pauses after an error, the pause doubles up to the maximum
|
||||
#define RECOVERY_RETRY_MIN_US 1000000
|
||||
#define RECOVERY_RETRY_MAX_US 60000000
|
||||
|
||||
//#define OSD_STUB
|
||||
|
||||
@@ -206,6 +209,7 @@ class osd_t
|
||||
pool_pg_num_t recovery_last_pg;
|
||||
object_id recovery_last_oid;
|
||||
int recovery_pg_done = 0, recovery_done = 0;
|
||||
int recovery_retry_timer_id = -1;
|
||||
osd_op_t *autosync_op = NULL;
|
||||
int autosync_copies_to_delete = 0;
|
||||
int autosync_timer_id = -1;
|
||||
@@ -336,6 +340,7 @@ class osd_t
|
||||
void submit_recovery_op(osd_recovery_op_t *op);
|
||||
void finish_recovery_op(osd_recovery_op_t *op);
|
||||
bool continue_recovery();
|
||||
void schedule_recovery_retry();
|
||||
pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg);
|
||||
|
||||
// scrub
|
||||
|
||||
@@ -249,12 +249,20 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
|
||||
return true;
|
||||
}
|
||||
|
||||
static uint64_t monotonic_us()
|
||||
{
|
||||
timespec tv;
|
||||
clock_gettime(CLOCK_MONOTONIC, &tv);
|
||||
return tv.tv_sec*1000000 + tv.tv_nsec/1000;
|
||||
}
|
||||
|
||||
bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
|
||||
{
|
||||
if (!pgs.size())
|
||||
{
|
||||
return false;
|
||||
}
|
||||
uint64_t now_us = monotonic_us();
|
||||
// Restart scanning from the same degraded/misplaced status as the last time
|
||||
for (int tried_degraded = 0; tried_degraded < 2; tried_degraded++)
|
||||
{
|
||||
@@ -268,7 +276,7 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
|
||||
for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++)
|
||||
{
|
||||
auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
|
||||
if ((pg_it->second.state & mask) == check && src.size() > 0)
|
||||
if ((pg_it->second.state & mask) == check && src.size() > 0 && pg_it->second.recovery_retry_us <= now_us)
|
||||
{
|
||||
auto pool_it = st_cli->pool_config.find(pg_it->first.pool_id);
|
||||
if (pool_it != st_cli->pool_config.end() && pool_it->second.backfillfull)
|
||||
@@ -332,19 +340,42 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
|
||||
{
|
||||
ringloop->set_immediate([this, op]()
|
||||
{
|
||||
auto pg_it = pgs.find((pool_pg_num_t){ .pool_id = INODE_POOL(op->oid.inode), .pg_num = map_to_pg(op->oid) });
|
||||
uint64_t now_us = monotonic_us();
|
||||
if (op->osd_op->reply.hdr.retval < 0)
|
||||
{
|
||||
// Error recovering object
|
||||
// EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not
|
||||
// and usually repeat (ENOSPC too), so pause the PG's recovery instead of spinning
|
||||
uint64_t retry_ms = 0;
|
||||
if (op->osd_op->reply.hdr.retval != -EPIPE && pg_it != pgs.end())
|
||||
{
|
||||
auto & pg = pg_it->second;
|
||||
if (pg.recovery_retry_us <= now_us)
|
||||
{
|
||||
pg.recovery_backoff_us = pg.recovery_backoff_us
|
||||
? std::min(pg.recovery_backoff_us*2, (uint64_t)RECOVERY_RETRY_MAX_US) : RECOVERY_RETRY_MIN_US;
|
||||
pg.recovery_retry_us = now_us + pg.recovery_backoff_us;
|
||||
}
|
||||
retry_ms = (pg.recovery_retry_us - now_us) / 1000;
|
||||
}
|
||||
printf(
|
||||
"[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n",
|
||||
"[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd, retrying in %ju ms\n",
|
||||
INODE_POOL(op->oid.inode), map_to_pg(op->oid),
|
||||
op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval
|
||||
op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval, retry_ms
|
||||
);
|
||||
}
|
||||
else if (log_level > 2)
|
||||
else
|
||||
{
|
||||
printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe);
|
||||
// An operation started before the pause doesn't end it
|
||||
if (pg_it != pgs.end() && pg_it->second.recovery_retry_us <= now_us)
|
||||
{
|
||||
pg_it->second.recovery_backoff_us = 0;
|
||||
}
|
||||
if (log_level > 2)
|
||||
{
|
||||
printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe);
|
||||
}
|
||||
}
|
||||
finish_recovery_op(op);
|
||||
});
|
||||
@@ -486,7 +517,37 @@ bool osd_t::continue_recovery()
|
||||
submit_recovery_op(&recovery_ops[op.oid]);
|
||||
}
|
||||
else
|
||||
{
|
||||
schedule_recovery_retry();
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// Wake recovery up when the nearest paused PG may be retried
|
||||
void osd_t::schedule_recovery_retry()
|
||||
{
|
||||
uint64_t now_us = monotonic_us(), retry_us = 0;
|
||||
for (auto & pgp: pgs)
|
||||
{
|
||||
if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us))
|
||||
{
|
||||
retry_us = pgp.second.recovery_retry_us;
|
||||
}
|
||||
}
|
||||
if (recovery_retry_timer_id >= 0)
|
||||
{
|
||||
tfd->clear_timer(recovery_retry_timer_id);
|
||||
recovery_retry_timer_id = -1;
|
||||
}
|
||||
if (retry_us)
|
||||
{
|
||||
recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id)
|
||||
{
|
||||
recovery_retry_timer_id = -1;
|
||||
peering_state = peering_state | OSD_RECOVERING;
|
||||
ringloop->wakeup();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -151,6 +151,7 @@ void osd_t::reset_pg(pg_t & pg)
|
||||
pg.degraded_objects.clear();
|
||||
pg.flush_actions.clear();
|
||||
pg.ver_override.clear();
|
||||
pg.recovery_retry_us = pg.recovery_backoff_us = 0;
|
||||
if (pg.flush_batch)
|
||||
{
|
||||
delete pg.flush_batch;
|
||||
|
||||
@@ -100,6 +100,8 @@ struct pg_t
|
||||
std::vector<osd_num_t> all_peers;
|
||||
// next scrub time
|
||||
uint64_t next_scrub = 0;
|
||||
// recovery is paused after errors until recovery_retry_us (CLOCK_MONOTONIC)
|
||||
uint64_t recovery_retry_us = 0, recovery_backoff_us = 0;
|
||||
bool history_changed = false;
|
||||
// peer list from the last peering event
|
||||
std::vector<osd_num_t> cur_peers;
|
||||
|
||||
@@ -161,6 +161,8 @@ OLD=1 SCHEME=xor ./test_enospc.sh
|
||||
OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh
|
||||
OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
|
||||
|
||||
./test_recovery_enospc.sh
|
||||
|
||||
./test_scrub.sh
|
||||
ZERO_OSD=2 ./test_scrub.sh
|
||||
SCHEME=xor ./test_scrub.sh
|
||||
|
||||
55
tests/test_recovery_enospc.sh
Executable file
55
tests/test_recovery_enospc.sh
Executable file
@@ -0,0 +1,55 @@
|
||||
#!/bin/bash -ex
|
||||
|
||||
# Recovery to a full OSD should pause the PG and retry later, not spin on ENOSPC
|
||||
|
||||
OSD_SIZE=128
|
||||
PG_COUNT=8
|
||||
PG_MINSIZE=1
|
||||
GLOBAL_CONFIG=',"client_retry_enospc":false,"osd_backfillfull_ratio":10'
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
pgs_on_osd3()
|
||||
{
|
||||
$ETCDCTL get /vitastor/pg/config --print-value-only | jq -s '[ .[0].items["1"][] | select(.osd_set | any(.[]; tostring == "3")) ] | length'
|
||||
}
|
||||
|
||||
pool_clean()
|
||||
{
|
||||
$ETCDCTL get /vitastor/pg/state/$1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$2
|
||||
}
|
||||
|
||||
# Pool 1 on OSD 1 and 2, pool 2 on OSD 3
|
||||
$ETCDCTL put /vitastor/config/osd/1 '{"tags":["a"]}'
|
||||
$ETCDCTL put /vitastor/config/osd/2 '{"tags":["a"]}'
|
||||
$ETCDCTL put /vitastor/config/osd/3 '{"tags":["b"]}'
|
||||
POOL1='"1":{'$POOLCFG',"pg_size":2,"pg_minsize":1,"pg_count":'$PG_COUNT
|
||||
POOL2='"2":{"name":"fill","scheme":"replicated","failure_domain":"osd","pg_size":1,"pg_minsize":1,"pg_count":1,"osd_tags":["b"]}'
|
||||
$ETCDCTL put /vitastor/config/pools '{'$POOL1',"osd_tags":["a"]},'$POOL2'}'
|
||||
wait_condition 60 '[[ $(pgs_on_osd3) -eq 0 ]] && pool_clean 1 $PG_COUNT && pool_clean 2 1' "Moving pool 1 off OSD 3"
|
||||
|
||||
$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=32M
|
||||
|
||||
# Fill OSD 3 up
|
||||
if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=2 -inode=1 -size=256M; then
|
||||
format_error "Should get ENOSPC, but didn't"
|
||||
fi
|
||||
|
||||
# Move pool 1 back to all OSDs, its recovery to OSD 3 now fails with ENOSPC
|
||||
$ETCDCTL put /vitastor/config/pools '{'$POOL1'},'$POOL2'}'
|
||||
wait_condition 60 '[[ $(pgs_on_osd3) -gt 0 ]]' "Moving pool 1 to OSD 3"
|
||||
wait_condition 30 "grep -q 'Recovery operation failed.*error -28' ./testdata/osd*.log" "Recovery ENOSPC"
|
||||
sleep 15
|
||||
FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed.*error -28')
|
||||
if [[ $FAILS -gt 100 ]]; then
|
||||
format_error "Recovery retried ENOSPC $FAILS times"
|
||||
fi
|
||||
if ! grep -q 'Recovery operation failed.*error -28, retrying in 4000 ms' ./testdata/osd*.log; then
|
||||
format_error "Recovery pause didn't grow"
|
||||
fi
|
||||
|
||||
# Free OSD 3, the paused recovery should resume and finish
|
||||
$VITASTOR_CLI rm-data --pool 2 --inode 1
|
||||
wait_condition 120 "pool_clean 1 $PG_COUNT" "Recovery after freeing space"
|
||||
|
||||
format_green OK
|
||||
Reference in New Issue
Block a user