osd: Pause recovery of a PG after an error instead of retrying it in a loop #143

Merged
vitalif merged 2 commits from koolkhel/vitastor:osd-recovery-error-backoff into master 2026-11-21 12:53:45 +03:00
8 changed files with 154 additions and 5 deletions

View File

@@ -2075,6 +2075,24 @@ jobs:
echo ""
done
test_recovery_enospc:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_recovery_enospc.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_scrub:
runs-on: ubuntu-latest
needs: build

View File

@@ -107,6 +107,11 @@ osd_t::~osd_t()
tfd->clear_timer(autosync_timer_id);
autosync_timer_id = -1;
}
if (recovery_retry_timer_id >= 0)
{
tfd->clear_timer(recovery_retry_timer_id);
recovery_retry_timer_id = -1;
}
ringloop->unregister_consumer(&consumer);
ringloop->unregister_consumer(&init_consumer);
// Tear down clients first: destroying a client cancels its operations and the

View File

@@ -38,6 +38,9 @@
#define DEFAULT_RECOVERY_QUEUE 1
#define DEFAULT_RECOVERY_PG_SWITCH 128
#define DEFAULT_RECOVERY_BATCH 16
// A PG's recovery pauses after an error, the pause doubles up to the maximum
#define RECOVERY_RETRY_MIN_US 1000000
#define RECOVERY_RETRY_MAX_US 60000000
//#define OSD_STUB
@@ -206,6 +209,7 @@ class osd_t
pool_pg_num_t recovery_last_pg;
object_id recovery_last_oid;
int recovery_pg_done = 0, recovery_done = 0;
int recovery_retry_timer_id = -1;
osd_op_t *autosync_op = NULL;
int autosync_copies_to_delete = 0;
int autosync_timer_id = -1;
@@ -336,6 +340,7 @@ class osd_t
void submit_recovery_op(osd_recovery_op_t *op);
void finish_recovery_op(osd_recovery_op_t *op);
bool continue_recovery();
void schedule_recovery_retry();
pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg);
// scrub

View File

@@ -249,12 +249,20 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t
return true;
}
static uint64_t monotonic_us()
{
timespec tv;
clock_gettime(CLOCK_MONOTONIC, &tv);
return tv.tv_sec*1000000 + tv.tv_nsec/1000;
}
bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
{
if (!pgs.size())
{
return false;
}
uint64_t now_us = monotonic_us();
// Restart scanning from the same degraded/misplaced status as the last time
for (int tried_degraded = 0; tried_degraded < 2; tried_degraded++)
{
@@ -268,7 +276,7 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op)
for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++)
{
auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects;
if ((pg_it->second.state & mask) == check && src.size() > 0)
if ((pg_it->second.state & mask) == check && src.size() > 0 && pg_it->second.recovery_retry_us <= now_us)
{
auto pool_it = st_cli->pool_config.find(pg_it->first.pool_id);
if (pool_it != st_cli->pool_config.end() && pool_it->second.backfillfull)
@@ -332,19 +340,42 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op)
{
ringloop->set_immediate([this, op]()
{
auto pg_it = pgs.find((pool_pg_num_t){ .pool_id = INODE_POOL(op->oid.inode), .pg_num = map_to_pg(op->oid) });
uint64_t now_us = monotonic_us();
if (op->osd_op->reply.hdr.retval < 0)
{
// Error recovering object
// EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not
// and usually repeat (ENOSPC too), so pause the PG's recovery instead of spinning
uint64_t retry_ms = 0;
if (op->osd_op->reply.hdr.retval != -EPIPE && pg_it != pgs.end())
{
auto & pg = pg_it->second;
if (pg.recovery_retry_us <= now_us)
{
pg.recovery_backoff_us = pg.recovery_backoff_us
? std::min(pg.recovery_backoff_us*2, (uint64_t)RECOVERY_RETRY_MAX_US) : RECOVERY_RETRY_MIN_US;
pg.recovery_retry_us = now_us + pg.recovery_backoff_us;
}
retry_ms = (pg.recovery_retry_us - now_us) / 1000;
}
printf(
"[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n",
"[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd, retrying in %ju ms\n",
INODE_POOL(op->oid.inode), map_to_pg(op->oid),
op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval
op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval, retry_ms
);
}
else if (log_level > 2)
else
{
printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe);
// An operation started before the pause doesn't end it
if (pg_it != pgs.end() && pg_it->second.recovery_retry_us <= now_us)
{
pg_it->second.recovery_backoff_us = 0;
}
if (log_level > 2)
{
printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe);
}
}
finish_recovery_op(op);
});
@@ -486,7 +517,37 @@ bool osd_t::continue_recovery()
submit_recovery_op(&recovery_ops[op.oid]);
}
else
{
schedule_recovery_retry();
return false;
}
}
return true;
}
// Wake recovery up when the nearest paused PG may be retried
void osd_t::schedule_recovery_retry()
{
uint64_t now_us = monotonic_us(), retry_us = 0;
for (auto & pgp: pgs)
{
if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us))
{
retry_us = pgp.second.recovery_retry_us;
}
}
if (recovery_retry_timer_id >= 0)
{
tfd->clear_timer(recovery_retry_timer_id);
recovery_retry_timer_id = -1;
}
if (retry_us)
{
recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id)
{
recovery_retry_timer_id = -1;
peering_state = peering_state | OSD_RECOVERING;
ringloop->wakeup();
});
}
}

View File

@@ -151,6 +151,7 @@ void osd_t::reset_pg(pg_t & pg)
pg.degraded_objects.clear();
pg.flush_actions.clear();
pg.ver_override.clear();
pg.recovery_retry_us = pg.recovery_backoff_us = 0;
if (pg.flush_batch)
{
delete pg.flush_batch;

View File

@@ -100,6 +100,8 @@ struct pg_t
std::vector<osd_num_t> all_peers;
// next scrub time
uint64_t next_scrub = 0;
// recovery is paused after errors until recovery_retry_us (CLOCK_MONOTONIC)
uint64_t recovery_retry_us = 0, recovery_backoff_us = 0;
bool history_changed = false;
// peer list from the last peering event
std::vector<osd_num_t> cur_peers;

View File

@@ -161,6 +161,8 @@ OLD=1 SCHEME=xor ./test_enospc.sh
OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh
OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh
./test_recovery_enospc.sh
./test_scrub.sh
ZERO_OSD=2 ./test_scrub.sh
SCHEME=xor ./test_scrub.sh

55
tests/test_recovery_enospc.sh Executable file
View File

@@ -0,0 +1,55 @@
#!/bin/bash -ex
# Recovery to a full OSD should pause the PG and retry later, not spin on ENOSPC
OSD_SIZE=128
PG_COUNT=8
PG_MINSIZE=1
GLOBAL_CONFIG=',"client_retry_enospc":false,"osd_backfillfull_ratio":10'
. `dirname $0`/run_3osds.sh
pgs_on_osd3()
{
$ETCDCTL get /vitastor/pg/config --print-value-only | jq -s '[ .[0].items["1"][] | select(.osd_set | any(.[]; tostring == "3")) ] | length'
}
pool_clean()
{
$ETCDCTL get /vitastor/pg/state/$1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$2
}
# Pool 1 on OSD 1 and 2, pool 2 on OSD 3
$ETCDCTL put /vitastor/config/osd/1 '{"tags":["a"]}'
$ETCDCTL put /vitastor/config/osd/2 '{"tags":["a"]}'
$ETCDCTL put /vitastor/config/osd/3 '{"tags":["b"]}'
POOL1='"1":{'$POOLCFG',"pg_size":2,"pg_minsize":1,"pg_count":'$PG_COUNT
POOL2='"2":{"name":"fill","scheme":"replicated","failure_domain":"osd","pg_size":1,"pg_minsize":1,"pg_count":1,"osd_tags":["b"]}'
$ETCDCTL put /vitastor/config/pools '{'$POOL1',"osd_tags":["a"]},'$POOL2'}'
wait_condition 60 '[[ $(pgs_on_osd3) -eq 0 ]] && pool_clean 1 $PG_COUNT && pool_clean 2 1' "Moving pool 1 off OSD 3"
$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=32M
# Fill OSD 3 up
if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=2 -inode=1 -size=256M; then
format_error "Should get ENOSPC, but didn't"
fi
# Move pool 1 back to all OSDs, its recovery to OSD 3 now fails with ENOSPC
$ETCDCTL put /vitastor/config/pools '{'$POOL1'},'$POOL2'}'
wait_condition 60 '[[ $(pgs_on_osd3) -gt 0 ]]' "Moving pool 1 to OSD 3"
wait_condition 30 "grep -q 'Recovery operation failed.*error -28' ./testdata/osd*.log" "Recovery ENOSPC"
sleep 15
FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed.*error -28')
if [[ $FAILS -gt 100 ]]; then
format_error "Recovery retried ENOSPC $FAILS times"
fi
if ! grep -q 'Recovery operation failed.*error -28, retrying in 4000 ms' ./testdata/osd*.log; then
format_error "Recovery pause didn't grow"
fi
# Free OSD 3, the paused recovery should resume and finish
$VITASTOR_CLI rm-data --pool 2 --inode 1
wait_condition 120 "pool_clean 1 $PG_COUNT" "Recovery after freeing space"
format_green OK