From 2edaa05bd0f9dac6e6178f8acb51b00fabaa762e Mon Sep 17 00:00:00 2001 From: Yury Luneff Date: Fri, 20 Nov 2026 16:28:58 +0300 Subject: [PATCH] osd: Wait for the old state lease to expire instead of exiting An OSD killed with SIGKILL or by a power loss can't revoke its etcd lease, so its /osd/state key stays until the lease expires. An OSD restarted before that printed "OSD N is still up" and exited, to be started again by systemd after one more full blockstore load. etcd also gives every lease a full TTL again when its leader changes, so after a power loss of the whole cluster most OSDs go through this, some of them several times. Check the state key before binding the sockets and acquiring the lease. While it exists and has a lease, check it again every etcd_quick_timeout (1 second by default). After 3 lease TTLs in total go on as before: create_osd_state() then finds the key and exits, because another OSD with this number really is up. The OSD doesn't listen and holds no lease while it waits, so it accepts no requests and writes nothing to etcd. --- .gitea/workflows/test.yml | 36 ++++++++++++++++++++++++++++ src/osd/osd.h | 2 ++ src/osd/osd_cluster.cpp | 50 +++++++++++++++++++++++++++++++++++---- tests/run_tests.sh | 3 +++ tests/test_osd_restart.sh | 41 ++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 4 deletions(-) create mode 100755 tests/test_osd_restart.sh diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ad781d0c..24aa48c6 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -1301,6 +1301,42 @@ jobs: echo "" done + test_osd_restart: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_osd_restart.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_osd_restart_antietcd: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: ANTIETCD=1 /root/vitastor/tests/test_osd_restart.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_switch_primary: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.h b/src/osd/osd.h index a48646fe..0bba1536 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -186,6 +186,7 @@ class osd_t osd_messenger_t msgr; bool use_perms = false; int etcd_failed_attempts = 0; + uint64_t etcd_lease_ttl = 0; std::string etcd_lease_id; json11::Json self_state; bool loading_peer_config = false; @@ -308,6 +309,7 @@ class osd_t json11::Json on_load_pgs_checks_hook(); void on_load_pgs_hook(bool success); void bind_socket(); + void wait_old_osd_state(uint64_t waited = 0); void acquire_lease(); json11::Json get_osd_state(); void create_osd_state(); diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index cdf82c75..5cf8f822 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -518,8 +518,7 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config) etcd_global_config_loaded = true; init_blockstore([this]() { - bind_socket(); - acquire_lease(); + wait_old_osd_state(); }); } @@ -530,14 +529,57 @@ void osd_t::on_reload_config_hook(json11::Json::object & global_config) renew_lease(true); } +// A killed OSD leaves its state key in etcd until its lease expires, and etcd restarts lease +// TTLs when its leader changes. So recheck the key for up to 3 TTLs, and only then start listening +void osd_t::wait_old_osd_state(uint64_t waited) +{ + std::string state_key = st_cli->etcd_prefix+"/osd/state/"+std::to_string(osd_num); + // Maximum lease TTL is (report interval) + retries * (timeout + repeat interval) + etcd_lease_ttl = etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000; + st_cli->etcd_txn(json11::Json::object { + { "success", json11::Json::array { + json11::Json::object { + { "request_range", json11::Json::object { + { "key", base64_encode(state_key) }, + } } + }, + } }, + }, st_cli->etcd_quick_timeout, 0, 0, [this, state_key, waited](std::string err, json11::Json data) + { + uint64_t max_wait = 3000*etcd_lease_ttl; + if (err != "") + { + printf("Error reading OSD state from etcd: %s, retrying\n", err.c_str()); + } + else if (data["responses"][0]["response_range"]["kvs"][0]["lease"].is_null() || waited >= max_wait) + { + // If the key is still there, create_osd_state() says that the OSD is up and exits + bind_socket(); + acquire_lease(); + return; + } + else if (!waited) + { + printf( + "[OSD %ju] Key %s already exists in etcd, waiting up to %ju seconds for its lease to expire\n", + this->osd_num, state_key.c_str(), max_wait/1000 + ); + } + uint64_t next = err != "" ? waited : waited+st_cli->etcd_quick_timeout; + tfd->set_timer(st_cli->etcd_quick_timeout, false, [this, next](int timer_id) + { + wait_old_osd_state(next); + }); + }); +} + // Acquire lease void osd_t::acquire_lease() { // Apply no_inode_stats before the first statistics report apply_no_inode_stats(); - // Maximum lease TTL is (report interval) + retries * (timeout + repeat interval) st_cli->etcd_call("/lease/grant", json11::Json::object { - { "TTL", etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000 } + { "TTL", etcd_lease_ttl } }, st_cli->etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data) { if (err != "" || data["ID"].string_value() == "") diff --git a/tests/run_tests.sh b/tests/run_tests.sh index caac41b7..68a82199 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -106,6 +106,9 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh ./test_root_node.sh +./test_osd_restart.sh +ANTIETCD=1 ./test_osd_restart.sh + ./test_switch_primary.sh ./test_write.sh diff --git a/tests/test_osd_restart.sh b/tests/test_osd_restart.sh new file mode 100755 index 00000000..76e1210d --- /dev/null +++ b/tests/test_osd_restart.sh @@ -0,0 +1,41 @@ +#!/bin/bash -ex + +. `dirname $0`/run_3osds.sh + +state_rev() +{ + $ETCDCTL get --write-out=json /vitastor/osd/state/1 | jq -r '.kvs[0].mod_revision' +} + +osd1_listens() +{ + ss -Hltnp | grep -q "pid=$OSD1_PID," +} + +# A killed OSD can't revoke its lease, so its state key outlives it. +# The restarted OSD has to wait for that lease to expire instead of exiting, +# and it must not listen until then +old_rev=$(state_rev) +kill -9 $OSD1_PID +start_osd 1 +wait_condition 30 "grep -q 'waiting .* for its lease to expire' ./testdata/osd1.log" "OSD 1 wait" +if osd1_listens; then + format_error "OSD 1 listens while it waits for the old lease" +fi +wait_condition 60 "[[ \$(state_rev) -gt $old_rev ]]" "OSD 1 restart" +new_rev=$(state_rev) +osd1_listens + +# A second OSD with the same number has to give up while the first one is alive +osd1_pid=$OSD1_PID +start_osd_on 1 $(osd_dev dup) +dup_pid=$OSD1_PID +OSD1_PID=$osd1_pid +wait_condition 120 "! kill -0 $dup_pid" "Second OSD 1 exit" +wait $dup_pid && format_error "Second OSD 1 exited with success" +kill -0 $OSD1_PID +if [[ $(state_rev) != $new_rev ]]; then + format_error "Second OSD 1 replaced the state of the first one" +fi + +format_green OK -- 2.47.3