diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 81e7c2a2..ad781d0c 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2075,6 +2075,24 @@ jobs: echo "" done + test_recovery_enospc: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_recovery_enospc.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_scrub: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.cpp b/src/osd/osd.cpp index 6932c30e..7f338b68 100644 --- a/src/osd/osd.cpp +++ b/src/osd/osd.cpp @@ -107,6 +107,11 @@ osd_t::~osd_t() tfd->clear_timer(autosync_timer_id); autosync_timer_id = -1; } + if (recovery_retry_timer_id >= 0) + { + tfd->clear_timer(recovery_retry_timer_id); + recovery_retry_timer_id = -1; + } ringloop->unregister_consumer(&consumer); ringloop->unregister_consumer(&init_consumer); // Tear down clients first: destroying a client cancels its operations and the diff --git a/src/osd/osd.h b/src/osd/osd.h index b146565c..a0ad4bd9 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -38,6 +38,9 @@ #define DEFAULT_RECOVERY_QUEUE 1 #define DEFAULT_RECOVERY_PG_SWITCH 128 #define DEFAULT_RECOVERY_BATCH 16 +// A PG's recovery pauses after an error, the pause doubles up to the maximum +#define RECOVERY_RETRY_MIN_US 1000000 +#define RECOVERY_RETRY_MAX_US 60000000 //#define OSD_STUB @@ -206,6 +209,7 @@ class osd_t pool_pg_num_t recovery_last_pg; object_id recovery_last_oid; int recovery_pg_done = 0, recovery_done = 0; + int recovery_retry_timer_id = -1; osd_op_t *autosync_op = NULL; int autosync_copies_to_delete = 0; int autosync_timer_id = -1; @@ -336,6 +340,7 @@ class osd_t void submit_recovery_op(osd_recovery_op_t *op); void finish_recovery_op(osd_recovery_op_t *op); bool continue_recovery(); + void schedule_recovery_retry(); pg_osd_set_state_t* change_osd_set(pg_osd_set_state_t *st, pg_t *pg); // scrub diff --git a/src/osd/osd_flush.cpp b/src/osd/osd_flush.cpp index 402ba0a4..5e6e38f3 100644 --- a/src/osd/osd_flush.cpp +++ b/src/osd/osd_flush.cpp @@ -249,12 +249,20 @@ bool osd_t::submit_flush_op(pool_id_t pool_id, pg_num_t pg_num, pg_flush_batch_t return true; } +static uint64_t monotonic_us() +{ + timespec tv; + clock_gettime(CLOCK_MONOTONIC, &tv); + return tv.tv_sec*1000000 + tv.tv_nsec/1000; +} + bool osd_t::pick_next_recovery(osd_recovery_op_t &op) { if (!pgs.size()) { return false; } + uint64_t now_us = monotonic_us(); // Restart scanning from the same degraded/misplaced status as the last time for (int tried_degraded = 0; tried_degraded < 2; tried_degraded++) { @@ -268,7 +276,7 @@ bool osd_t::pick_next_recovery(osd_recovery_op_t &op) for (auto pg_it = pgs.lower_bound(recovery_last_pg); pg_it != pgs.end(); pg_it++) { auto & src = recovery_last_degraded ? pg_it->second.degraded_objects : pg_it->second.misplaced_objects; - if ((pg_it->second.state & mask) == check && src.size() > 0) + if ((pg_it->second.state & mask) == check && src.size() > 0 && pg_it->second.recovery_retry_us <= now_us) { auto pool_it = st_cli->pool_config.find(pg_it->first.pool_id); if (pool_it != st_cli->pool_config.end() && pool_it->second.backfillfull) @@ -332,19 +340,42 @@ void osd_t::submit_recovery_op(osd_recovery_op_t *op) { ringloop->set_immediate([this, op]() { + auto pg_it = pgs.find((pool_pg_num_t){ .pool_id = INODE_POOL(op->oid.inode), .pg_num = map_to_pg(op->oid) }); + uint64_t now_us = monotonic_us(); if (op->osd_op->reply.hdr.retval < 0) { // Error recovering object // EPIPE is totally harmless (peer is gone), others like EIO/EDOM may be not + // and usually repeat (ENOSPC too), so pause the PG's recovery instead of spinning + uint64_t retry_ms = 0; + if (op->osd_op->reply.hdr.retval != -EPIPE && pg_it != pgs.end()) + { + auto & pg = pg_it->second; + if (pg.recovery_retry_us <= now_us) + { + pg.recovery_backoff_us = pg.recovery_backoff_us + ? std::min(pg.recovery_backoff_us*2, (uint64_t)RECOVERY_RETRY_MAX_US) : RECOVERY_RETRY_MIN_US; + pg.recovery_retry_us = now_us + pg.recovery_backoff_us; + } + retry_ms = (pg.recovery_retry_us - now_us) / 1000; + } printf( - "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd\n", + "[PG %u/%u] Recovery operation failed with object %jx:%jx: error %jd, retrying in %ju ms\n", INODE_POOL(op->oid.inode), map_to_pg(op->oid), - op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval + op->oid.inode, op->oid.stripe, op->osd_op->reply.hdr.retval, retry_ms ); } - else if (log_level > 2) + else { - printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe); + // An operation started before the pause doesn't end it + if (pg_it != pgs.end() && pg_it->second.recovery_retry_us <= now_us) + { + pg_it->second.recovery_backoff_us = 0; + } + if (log_level > 2) + { + printf("Recovery operation done for %jx:%jx\n", op->oid.inode, op->oid.stripe); + } } finish_recovery_op(op); }); @@ -486,7 +517,37 @@ bool osd_t::continue_recovery() submit_recovery_op(&recovery_ops[op.oid]); } else + { + schedule_recovery_retry(); return false; + } } return true; } + +// Wake recovery up when the nearest paused PG may be retried +void osd_t::schedule_recovery_retry() +{ + uint64_t now_us = monotonic_us(), retry_us = 0; + for (auto & pgp: pgs) + { + if (pgp.second.recovery_retry_us > now_us && (!retry_us || retry_us > pgp.second.recovery_retry_us)) + { + retry_us = pgp.second.recovery_retry_us; + } + } + if (recovery_retry_timer_id >= 0) + { + tfd->clear_timer(recovery_retry_timer_id); + recovery_retry_timer_id = -1; + } + if (retry_us) + { + recovery_retry_timer_id = tfd->set_timer_us(retry_us - now_us, false, [this](int timer_id) + { + recovery_retry_timer_id = -1; + peering_state = peering_state | OSD_RECOVERING; + ringloop->wakeup(); + }); + } +} diff --git a/src/osd/osd_peering.cpp b/src/osd/osd_peering.cpp index 1a88da05..0b9a8e06 100644 --- a/src/osd/osd_peering.cpp +++ b/src/osd/osd_peering.cpp @@ -151,6 +151,7 @@ void osd_t::reset_pg(pg_t & pg) pg.degraded_objects.clear(); pg.flush_actions.clear(); pg.ver_override.clear(); + pg.recovery_retry_us = pg.recovery_backoff_us = 0; if (pg.flush_batch) { delete pg.flush_batch; diff --git a/src/osd/osd_peering_pg.h b/src/osd/osd_peering_pg.h index cdbd91ef..0225f085 100644 --- a/src/osd/osd_peering_pg.h +++ b/src/osd/osd_peering_pg.h @@ -100,6 +100,8 @@ struct pg_t std::vector all_peers; // next scrub time uint64_t next_scrub = 0; + // recovery is paused after errors until recovery_retry_us (CLOCK_MONOTONIC) + uint64_t recovery_retry_us = 0, recovery_backoff_us = 0; bool history_changed = false; // peer list from the last peering event std::vector cur_peers; diff --git a/tests/run_tests.sh b/tests/run_tests.sh index dc4c6577..caac41b7 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -161,6 +161,8 @@ OLD=1 SCHEME=xor ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 ./test_enospc.sh OLD=1 IMMEDIATE_COMMIT=1 SCHEME=xor ./test_enospc.sh +./test_recovery_enospc.sh + ./test_scrub.sh ZERO_OSD=2 ./test_scrub.sh SCHEME=xor ./test_scrub.sh diff --git a/tests/test_recovery_enospc.sh b/tests/test_recovery_enospc.sh new file mode 100755 index 00000000..7fae381f --- /dev/null +++ b/tests/test_recovery_enospc.sh @@ -0,0 +1,55 @@ +#!/bin/bash -ex + +# Recovery to a full OSD should pause the PG and retry later, not spin on ENOSPC + +OSD_SIZE=128 +PG_COUNT=8 +PG_MINSIZE=1 +GLOBAL_CONFIG=',"client_retry_enospc":false,"osd_backfillfull_ratio":10' + +. `dirname $0`/run_3osds.sh + +pgs_on_osd3() +{ + $ETCDCTL get /vitastor/pg/config --print-value-only | jq -s '[ .[0].items["1"][] | select(.osd_set | any(.[]; tostring == "3")) ] | length' +} + +pool_clean() +{ + $ETCDCTL get /vitastor/pg/state/$1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$2 +} + +# Pool 1 on OSD 1 and 2, pool 2 on OSD 3 +$ETCDCTL put /vitastor/config/osd/1 '{"tags":["a"]}' +$ETCDCTL put /vitastor/config/osd/2 '{"tags":["a"]}' +$ETCDCTL put /vitastor/config/osd/3 '{"tags":["b"]}' +POOL1='"1":{'$POOLCFG',"pg_size":2,"pg_minsize":1,"pg_count":'$PG_COUNT +POOL2='"2":{"name":"fill","scheme":"replicated","failure_domain":"osd","pg_size":1,"pg_minsize":1,"pg_count":1,"osd_tags":["b"]}' +$ETCDCTL put /vitastor/config/pools '{'$POOL1',"osd_tags":["a"]},'$POOL2'}' +wait_condition 60 '[[ $(pgs_on_osd3) -eq 0 ]] && pool_clean 1 $PG_COUNT && pool_clean 2 1' "Moving pool 1 off OSD 3" + +$VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=1 -inode=1 -size=32M + +# Fill OSD 3 up +if $VITASTOR_FIO -bs=1M -direct=1 -iodepth=4 -rw=write -pool=2 -inode=1 -size=256M; then + format_error "Should get ENOSPC, but didn't" +fi + +# Move pool 1 back to all OSDs, its recovery to OSD 3 now fails with ENOSPC +$ETCDCTL put /vitastor/config/pools '{'$POOL1'},'$POOL2'}' +wait_condition 60 '[[ $(pgs_on_osd3) -gt 0 ]]' "Moving pool 1 to OSD 3" +wait_condition 30 "grep -q 'Recovery operation failed.*error -28' ./testdata/osd*.log" "Recovery ENOSPC" +sleep 15 +FAILS=$(cat ./testdata/osd*.log | grep -c 'Recovery operation failed.*error -28') +if [[ $FAILS -gt 100 ]]; then + format_error "Recovery retried ENOSPC $FAILS times" +fi +if ! grep -q 'Recovery operation failed.*error -28, retrying in 4000 ms' ./testdata/osd*.log; then + format_error "Recovery pause didn't grow" +fi + +# Free OSD 3, the paused recovery should resume and finish +$VITASTOR_CLI rm-data --pool 2 --inode 1 +wait_condition 120 "pool_clean 1 $PG_COUNT" "Recovery after freeing space" + +format_green OK