diff --git a/src/osd/osd.h b/src/osd/osd.h index a48646fe..e940f7d4 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -354,14 +354,14 @@ class osd_t void submit_flush_batch(pg_t & pg); void handle_flush_op(bool rollback, pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, osd_num_t peer_osd, int retval); bool submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t *fb, bool rollback, osd_num_t peer_osd, int count, obj_ver_id *data); - bool pick_next_recovery(osd_recovery_op_t &op); + bool pick_next_recovery(osd_recovery_op_t &op, uint64_t now_us); void submit_recovery_op(osd_recovery_op_t *op); void finish_recovery_op(osd_recovery_op_t *op); void continue_primary_forget(osd_op_t *cur_op); void handle_forget_subop(osd_recovery_op_t *op, osd_op_t *sub, bool failed); void submit_forget_subop(object_id oid, uint64_t version, osd_num_t osd, osd_recovery_op_t *op); bool continue_recovery(); - void schedule_recovery_retry(); + void schedule_recovery_retry(uint64_t now_us); pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg); // scrub diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 04c923f5..085cb363 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -264,13 +264,12 @@ static uint64_t monotonic_us() return tv.tv_sec*1000000 + tv.tv_nsec/1000; } -bool osd_t::pick_next_recovery(osd_recovery_op_t &op) +bool osd_t::pick_next_recovery(osd_recovery_op_t &op, uint64_t now_us) { if (!pgs.size()) { return false; } - uint64_t now_us = monotonic_us(); // Cycle through degraded (recovery), misplaced (rebalance) and tombstone // cleanup, restarting the scan from the kind handled last time for (int tried = 0; tried < OSD_REC_MAX; tried++) @@ -563,17 +562,19 @@ void osd_t::tune_recovery() // Just trigger write requests for degraded objects. They'll be recovered during writing bool osd_t::continue_recovery() { + // One time for picking and the retry timer, or a pause ending in between is lost + uint64_t now_us = monotonic_us(); while (recovery_ops.size() < recovery_queue_depth) { osd_recovery_op_t op; - if (pick_next_recovery(op)) + if (pick_next_recovery(op, now_us)) { recovery_ops[op.oid] = op; submit_recovery_op(&recovery_ops[op.oid]); } else { - schedule_recovery_retry(); + schedule_recovery_retry(now_us); return false; } } @@ -581,9 +582,9 @@ bool osd_t::continue_recovery() } // Wake recovery up when the nearest paused PG may be retried -void osd_t::schedule_recovery_retry() +void osd_t::schedule_recovery_retry(uint64_t now_us) { - uint64_t now_us = monotonic_us(), retry_us = 0; + uint64_t retry_us = 0; for (auto & pgp: pgs) { if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us)) @@ -598,7 +599,8 @@ void osd_t::schedule_recovery_retry() } if (retry_us) { - recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id) + uint64_t cur_us = monotonic_us(); + recovery_retry_timer_id = tfd->set_timer_us(retry_us > cur_us ? retry_us - cur_us : 1, false, [this](int timer_id) { recovery_retry_timer_id = -1; peering_state = peering_state | OSD_RECOVERING;