From 545f4ad05df14ecad76b8dd5393b63664e471a83 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Sat, 21 Nov 2026 11:49:59 +0300 Subject: [PATCH 1/2] osd: Pause recovery of a PG after an error instead of retrying it in a loop A recovery operation that fails with an error that doesn't drop the connection (ENOSPC from a full target OSD, EIO, EDOM) was retried at once, forever: finish_recovery_op() restarts recovery immediately and the object stays degraded or misplaced. When the monitor doesn't mark the pool backfillfull, for example because the OSD is filled with data it doesn't account for, the primary spins at thousands of failures per second, each logging two lines. Now a failed recovery operation (except EPIPE) pauses recovery of its PG for 1 second, doubling up to 60 seconds while it keeps failing, and a successful one resets the pause. pick_next_recovery() skips paused PGs and a timer resumes recovery when the nearest pause ends. Re-peering clears it. --- .gitea/workflows/test.yml | 18 +++++++++ src/osd/osd.cpp | 5 +++ src/osd/osd.h | 5 +++ src/osd/osd_flush.cpp | 69 ++++++++++++++++++++++++++++++++--- src/osd/osd_peering.cpp | 1 + src/osd/osd_peering_pg.h | 2 + tests/run_tests.sh | 2 + tests/test_recovery_enospc.sh | 52 ++++++++++++++++++++++++++ 8 files changed, 149 insertions(+), 5 deletions(-) create mode 100755 tests/test_recovery_enospc.sh diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 81e7c2a2..ad781d0c 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2075,6 +2075,24 @@ jobs: echo "" done + test_recovery_enospc: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_recovery_enospc.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 6932c30e..7f338b68 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -107,6 +107,11 @@ osd_t::~osd_t() tfd->clear_timer(autosync_timer_id); autosync_timer_id = -1; } + if (recovery_retry_timer_id >= 0) + { + tfd->clear_timer(recovery_retry_timer_id); + recovery_retry_timer_id = -1; + } ringloop->unregister_consumer(&consumer); ringloop->unregister_consumer(&init_consumer); // Tear down clients first: destroying a client cancels its operations and the diff --git a/src/osd/osd.h b/src/osd/osd.h index b146565c..a0ad4bd9 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -38,6 +38,9 @@ #define DEFAULT_RECOVERY_QUEUE 1 #define DEFAULT_RECOVERY_PG_SWITCH 128 #define DEFAULT_RECOVERY_BATCH 16 +// A PG's recovery pauses after an error, the pause doubles up to the maximum +#define RECOVERY_RETRY_MIN_US 1000000 +#define RECOVERY_RETRY_MAX_US 60000000 //#define OSD_STUB @@ -206,6 +209,7 @@ class osd_t pool_pg_num_t recovery_last_pg; object_id recovery_last_oid; int recovery_pg_done = 0, recovery_done = 0; + int recovery_retry_timer_id = -1; osd_op_t *autosync_op = NULL; int autosync_copies_to_delete = 0; int autosync_timer_id = -1; @@ -336,6 +340,7 @@ class osd_t void submit_recovery_op(osd_recovery_op_t *op); void finish_recovery_op(osd_recovery_op_t *op); bool continue_recovery(); + void schedule_recovery_retry(); pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg); // scrub diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 402ba0a4..eb62ae31 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -249,12 +249,20 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t return true; } +static uint64_t monotonic_us() +{ + timespec tv; + clock_gettime(CLOCK_MONOTONIC, &tv); + return tv.tv_sec*1000000 + tv.tv_nsec/1000; +} + bool osd_t::pick_next_recovery(osd_recovery_op_t &op) { if (!pgs.size()) { return false; } + uint64_t now_us = monotonic_us(); // Restart scanning from the same degraded/misplaced status as the last time for (int tried_degraded = 0; tried_degraded < 2; tried_degraded++) { @@ -268,7 +276,7 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op) for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++) { auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects; - if ((pg_it->second.state & mask) == check && src.size() > 0) + if ((pg_it->second.state & mask) == check && src.size() > 0 && pg_it->second.recovery_retry_us <= now_us) { auto pool_it = st_cli->pool_config.find(pg_it->first.pool_id); if (pool_it != st_cli->pool_config.end() && pool_it->second.backfillfull) @@ -332,19 +340,40 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op) { ringloop->set_immediate([this, op]() { + auto pg_it = pgs.find((pool_pg_num_t){ .pool_id = INODE_POOL(op->oid.inode), .pg_num = map_to_pg(op->oid) }); if (op->osd_op->reply.hdr.retval < 0) { // Error recovering object // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not + // and usually repeat (ENOSPC too), so pause the PG's recovery instead of spinning + uint64_t now_us = monotonic_us(), retry_ms = 0; + if (op->osd_op->reply.hdr.retval != -EPIPE && pg_it != pgs.end()) + { + auto & pg = pg_it->second; + if (pg.recovery_retry_us <= now_us) + { + pg.recovery_backoff_us = pg.recovery_backoff_us + ? std::min(pg.recovery_backoff_us*2, (uint64_t)RECOVERY_RETRY_MAX_US) : RECOVERY_RETRY_MIN_US; + pg.recovery_retry_us = now_us + pg.recovery_backoff_us; + } + retry_ms = (pg.recovery_retry_us - now_us) / 1000; + } printf( - "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n", + "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd, retrying in %ju ms\n", INODE_POOL(op->oid.inode), map_to_pg(op->oid), - op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval + op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval, retry_ms ); } - else if (log_level > 2) + else { - printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe); + if (pg_it != pgs.end()) + { + pg_it->second.recovery_backoff_us = 0; + } + if (log_level > 2) + { + printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe); + } } finish_recovery_op(op); }); @@ -486,7 +515,37 @@ bool osd_t::continue_recovery() submit_recovery_op(&recovery_ops[op.oid]); } else + { + schedule_recovery_retry(); return false; + } } return true; } + +// Wake recovery up when the nearest paused PG may be retried +void osd_t::schedule_recovery_retry() +{ + uint64_t now_us = monotonic_us(), retry_us = 0; + for (auto & pgp: pgs) + { + if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us)) + { + retry_us = pgp.second.recovery_retry_us; + } + } + if (recovery_retry_timer_id >= 0) + { + tfd->clear_timer(recovery_retry_timer_id); + recovery_retry_timer_id = -1; + } + if (retry_us) + { + recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id) + { + recovery_retry_timer_id = -1; + peering_state = peering_state | OSD_RECOVERING; + ringloop->wakeup(); + }); + } +} diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index 1a88da05..0b9a8e06 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -151,6 +151,7 @@ void osd_t::reset_pg(pg_t & pg) pg.degraded_objects.clear(); pg.flush_actions.clear(); pg.ver_override.clear(); + pg.recovery_retry_us = pg.recovery_backoff_us = 0; if (pg.flush_batch) { delete pg.flush_batch; diff --git a/src/osd/osd_peering_pg.h b/src/osd/osd_peering_pg.h index cdbd91ef..0225f085 100644 --- a/src/osd/osd_peering_pg.h +++ b/src/osd/osd_peering_pg.h @@ -100,6 +100,8 @@ struct pg_t std::vector all_peers; // next scrub time uint64_t next_scrub = 0; + // recovery is paused after errors until recovery_retry_us (CLOCK_MONOTONIC) + uint64_t recovery_retry_us = 0, recovery_backoff_us = 0; bool history_changed = false; // peer list from the last peering event std::vector cur_peers; diff --git a/tests/run_tests.sh b/tests/run_tests.sh index dc4c6577..caac41b7 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -161,6 +161,8 @@ OLD=1 SCHEME=xor ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh +./test_recovery_enospc.sh + ./test_scrub.sh ZERO_OSD=2 ./test_scrub.sh SCHEME=xor ./test_scrub.sh diff --git a/tests/test_recovery_enospc.sh b/tests/test_recovery_enospc.sh new file mode 100755 index 00000000..c6c816ee --- /dev/null +++ b/tests/test_recovery_enospc.sh @@ -0,0 +1,52 @@ +#!/bin/bash -ex + +# Recovery to a full OSD should pause the PG and retry later, not spin on ENOSPC + +OSD_SIZE=128 +PG_COUNT=8 +PG_MINSIZE=1 +GLOBAL_CONFIG=',"client_retry_enospc":false,"osd_backfillfull_ratio":10' + +. `dirname $0`/run_3osds.sh + +pgs_on_osd3() +{ + $ETCDCTL get /vitastor/pg/config --print-value-only | jq -s '[ .[0].items["1"][] | select(.osd_set | any(.[]; tostring == "3")) ] | length' +} + +pool_clean() +{ + $ETCDCTL get /vitastor/pg/state/$1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$2 +} + +# Pool 1 on OSD 1 and 2, pool 2 on OSD 3 +$ETCDCTL put /vitastor/config/osd/1 '{"tags":["a"]}' +$ETCDCTL put /vitastor/config/osd/2 '{"tags":["a"]}' +$ETCDCTL put /vitastor/config/osd/3 '{"tags":["b"]}' +POOL1='"1":{'$POOLCFG',"pg_size":2,"pg_minsize":1,"pg_count":'$PG_COUNT +POOL2='"2":{"name":"fill","scheme":"replicated","failure_domain":"osd","pg_size":1,"pg_minsize":1,"pg_count":1,"osd_tags":["b"]}' +$ETCDCTL put /vitastor/config/pools '{'$POOL1',"osd_tags":["a"]},'$POOL2'}' +wait_condition 60 '[[ $(pgs_on_osd3) -eq 0 ]] && pool_clean 1 $PG_COUNT && pool_clean 2 1' "Moving pool 1 off OSD 3" + +$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=32M + +# Fill OSD 3 up +if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=2 -inode=1 -size=256M; then + format_error "Should get ENOSPC, but didn't" +fi + +# Move pool 1 back to all OSDs, its recovery to OSD 3 now fails with ENOSPC +$ETCDCTL put /vitastor/config/pools '{'$POOL1'},'$POOL2'}' +wait_condition 60 '[[ $(pgs_on_osd3) -gt 0 ]]' "Moving pool 1 to OSD 3" +wait_condition 30 "grep -q 'Recovery operation failed.*error -28' ./testdata/osd*.log" "Recovery ENOSPC" +sleep 15 +FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed') +if [[ $FAILS -gt 100 ]]; then + format_error "Recovery retried ENOSPC $FAILS times" +fi + +# Free OSD 3, the paused recovery should resume and finish +$VITASTOR_CLI rm-data --pool 2 --inode 1 +wait_condition 120 "pool_clean 1 $PG_COUNT" "Recovery after freeing space" + +format_green OK -- 2.47.3 From 69756095bae79e828cd7ecccd3f79e4e53b3af0a Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Sat, 21 Nov 2026 12:30:26 +0300 Subject: [PATCH 2/2] osd: Keep the recovery pause on successes of operations started before it With recovery_queue_depth > 1, an operation submitted before a failure could succeed during the pause and reset it, so the pause never grew past 1 second. Reset it only on a success after the pause ends. Count only ENOSPC failures in test_recovery_enospc and check that the pause grows. --- src/osd/osd_flush.cpp | 6 ++++-- tests/test_recovery_enospc.sh | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index eb62ae31..5e6e38f3 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -341,12 +341,13 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op) ringloop->set_immediate([this, op]() { auto pg_it = pgs.find((pool_pg_num_t){ .pool_id = INODE_POOL(op->oid.inode), .pg_num = map_to_pg(op->oid) }); + uint64_t now_us = monotonic_us(); if (op->osd_op->reply.hdr.retval < 0) { // Error recovering object // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not // and usually repeat (ENOSPC too), so pause the PG's recovery instead of spinning - uint64_t now_us = monotonic_us(), retry_ms = 0; + uint64_t retry_ms = 0; if (op->osd_op->reply.hdr.retval != -EPIPE && pg_it != pgs.end()) { auto & pg = pg_it->second; @@ -366,7 +367,8 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op) } else { - if (pg_it != pgs.end()) + // An operation started before the pause doesn't end it + if (pg_it != pgs.end() && pg_it->second.recovery_retry_us <= now_us) { pg_it->second.recovery_backoff_us = 0; } diff --git a/tests/test_recovery_enospc.sh b/tests/test_recovery_enospc.sh index c6c816ee..7fae381f 100755 --- a/tests/test_recovery_enospc.sh +++ b/tests/test_recovery_enospc.sh @@ -40,10 +40,13 @@ $ETCDCTL put /vitastor/config/pools '{'$POOL1'},'$POOL2'}' wait_condition 60 '[[ $(pgs_on_osd3) -gt 0 ]]' "Moving pool 1 to OSD 3" wait_condition 30 "grep -q 'Recovery operation failed.*error -28' ./testdata/osd*.log" "Recovery ENOSPC" sleep 15 -FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed') +FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed.*error -28') if [[ $FAILS -gt 100 ]]; then format_error "Recovery retried ENOSPC $FAILS times" fi +if ! grep -q 'Recovery operation failed.*error -28, retrying in 4000 ms' ./testdata/osd*.log; then + format_error "Recovery pause didn't grow" +fi # Free OSD 3, the paused recovery should resume and finish $VITASTOR_CLI rm-data --pool 2 --inode 1 -- 2.47.3