From 332c06989f8555f8cf2ad439113db3c76bda0f49 Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Sat, 21 Nov 2026 17:35:38 +0300 Subject: [PATCH] osd: Use one timestamp for picking recovery and arming its retry timer pick_next_recovery() and schedule_recovery_retry() each read the clock. A PG pause that ended between the two reads was skipped by the pick but not counted by the timer, so no timer was armed and the PG's recovery waited for an unrelated event. Take the time once in continue_recovery() and arm the timer for at least 1 us. --- src/osd/osd.h | 4 ++-- src/osd/osd_flush.cpp | 16 +++++++++------- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/osd/osd.h b/src/osd/osd.h index a48646fe..e940f7d4 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -354,14 +354,14 @@ class osd_t void submit_flush_batch(pg_t & pg); void handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval); bool submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t peer_osd, int count, obj_ver_id *data); - bool pick_next_recovery(osd_recovery_op_t &op); + bool pick_next_recovery(osd_recovery_op_t &op, uint64_t now_us); void submit_recovery_op(osd_recovery_op_t *op); void finish_recovery_op(osd_recovery_op_t *op); void continue_primary_forget(osd_op_t *cur_op); void handle_forget_subop(osd_recovery_op_t *op, osd_op_t *sub, bool failed); void submit_forget_subop(object_id oid, uint64_t version, osd_num_t osd, osd_recovery_op_t *op); bool continue_recovery(); - void schedule_recovery_retry(); + void schedule_recovery_retry(uint64_t now_us); pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg); // scrub diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 04c923f5..085cb363 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -264,13 +264,12 @@ static uint64_t monotonic_us() return tv.tv_sec*1000000 + tv.tv_nsec/1000; } -bool osd_t::pick_next_recovery(osd_recovery_op_t &op) +bool osd_t::pick_next_recovery(osd_recovery_op_t &op, uint64_t now_us) { if (!pgs.size()) { return false; } - uint64_t now_us = monotonic_us(); // Cycle through degraded (recovery), misplaced (rebalance) and tombstone // cleanup, restarting the scan from the kind handled last time for (int tried = 0; tried < OSD_REC_MAX; tried++) @@ -563,17 +562,19 @@ void osd_t::tune_recovery() // Just trigger write requests for degraded objects. They'll be recovered during writing bool osd_t::continue_recovery() { + // One time for picking and the retry timer, or a pause ending in between is lost + uint64_t now_us = monotonic_us(); while (recovery_ops.size() < recovery_queue_depth) { osd_recovery_op_t op; - if (pick_next_recovery(op)) + if (pick_next_recovery(op, now_us)) { recovery_ops[op.oid] = op; submit_recovery_op(&recovery_ops[op.oid]); } else { - schedule_recovery_retry(); + schedule_recovery_retry(now_us); return false; } } @@ -581,9 +582,9 @@ bool osd_t::continue_recovery() } // Wake recovery up when the nearest paused PG may be retried -void osd_t::schedule_recovery_retry() +void osd_t::schedule_recovery_retry(uint64_t now_us) { - uint64_t now_us = monotonic_us(), retry_us = 0; + uint64_t retry_us = 0; for (auto & pgp: pgs) { if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us)) @@ -598,7 +599,8 @@ void osd_t::schedule_recovery_retry() } if (retry_us) { - recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id) + uint64_t cur_us = monotonic_us(); + recovery_retry_timer_id = tfd->set_timer_us(retry_us > cur_us ? retry_us - cur_us : 1, false, [this](int timer_id) { recovery_retry_timer_id = -1; peering_state = peering_state | OSD_RECOVERING; -- 2.47.3