diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index ad781d0c..24aa48c6 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -1301,6 +1301,42 @@ jobs: echo "" done + test_osd_restart: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: /root/vitastor/tests/test_osd_restart.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + + test_osd_restart_antietcd: + runs-on: ubuntu-latest + needs: build + container: ${{env.TEST_IMAGE}}:${{github.sha}} + steps: + - name: Run test + id: test + timeout-minutes: 3 + run: ANTIETCD=1 /root/vitastor/tests/test_osd_restart.sh + - name: Print logs + if: always() && steps.test.outcome == 'failure' + run: | + for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do + echo "-------- $i --------" + cat $i + echo "" + done + test_switch_primary: runs-on: ubuntu-latest needs: build diff --git a/src/osd/osd.h b/src/osd/osd.h index a48646fe..0bba1536 100644 --- a/src/osd/osd.h +++ b/src/osd/osd.h @@ -186,6 +186,7 @@ class osd_t osd_messenger_t msgr; bool use_perms = false; int etcd_failed_attempts = 0; + uint64_t etcd_lease_ttl = 0; std::string etcd_lease_id; json11::Json self_state; bool loading_peer_config = false; @@ -308,6 +309,7 @@ class osd_t json11::Json on_load_pgs_checks_hook(); void on_load_pgs_hook(bool success); void bind_socket(); + void wait_old_osd_state(uint64_t waited = 0); void acquire_lease(); json11::Json get_osd_state(); void create_osd_state(); diff --git a/src/osd/osd_cluster.cpp b/src/osd/osd_cluster.cpp index cdf82c75..5cf8f822 100644 --- a/src/osd/osd_cluster.cpp +++ b/src/osd/osd_cluster.cpp @@ -518,8 +518,7 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config) etcd_global_config_loaded = true; init_blockstore([this]() { - bind_socket(); - acquire_lease(); + wait_old_osd_state(); }); } @@ -530,14 +529,57 @@ void osd_t::on_reload_config_hook(json11::Json::object & global_config) renew_lease(true); } +// A killed OSD leaves its state key in etcd until its lease expires, and etcd restarts lease +// TTLs when its leader changes. So recheck the key for up to 3 TTLs, and only then start listening +void osd_t::wait_old_osd_state(uint64_t waited) +{ + std::string state_key = st_cli->etcd_prefix+"/osd/state/"+std::to_string(osd_num); + // Maximum lease TTL is (report interval) + retries * (timeout + repeat interval) + etcd_lease_ttl = etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000; + st_cli->etcd_txn(json11::Json::object { + { "success", json11::Json::array { + json11::Json::object { + { "request_range", json11::Json::object { + { "key", base64_encode(state_key) }, + } } + }, + } }, + }, st_cli->etcd_quick_timeout, 0, 0, [this, state_key, waited](std::string err, json11::Json data) + { + uint64_t max_wait = 3000*etcd_lease_ttl; + if (err != "") + { + printf("Error reading OSD state from etcd: %s, retrying\n", err.c_str()); + } + else if (data["responses"][0]["response_range"]["kvs"][0]["lease"].is_null() || waited >= max_wait) + { + // If the key is still there, create_osd_state() says that the OSD is up and exits + bind_socket(); + acquire_lease(); + return; + } + else if (!waited) + { + printf( + "[OSD %ju] Key %s already exists in etcd, waiting up to %ju seconds for its lease to expire\n", + this->osd_num, state_key.c_str(), max_wait/1000 + ); + } + uint64_t next = err != "" ? waited : waited+st_cli->etcd_quick_timeout; + tfd->set_timer(st_cli->etcd_quick_timeout, false, [this, next](int timer_id) + { + wait_old_osd_state(next); + }); + }); +} + // Acquire lease void osd_t::acquire_lease() { // Apply no_inode_stats before the first statistics report apply_no_inode_stats(); - // Maximum lease TTL is (report interval) + retries * (timeout + repeat interval) st_cli->etcd_call("/lease/grant", json11::Json::object { - { "TTL", etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000 } + { "TTL", etcd_lease_ttl } }, st_cli->etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data) { if (err != "" || data["ID"].string_value() == "") diff --git a/tests/run_tests.sh b/tests/run_tests.sh index caac41b7..68a82199 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -106,6 +106,9 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh ./test_root_node.sh +./test_osd_restart.sh +ANTIETCD=1 ./test_osd_restart.sh + ./test_switch_primary.sh ./test_write.sh diff --git a/tests/test_osd_restart.sh b/tests/test_osd_restart.sh new file mode 100755 index 00000000..76e1210d --- /dev/null +++ b/tests/test_osd_restart.sh @@ -0,0 +1,41 @@ +#!/bin/bash -ex + +. `dirname $0`/run_3osds.sh + +state_rev() +{ + $ETCDCTL get --write-out=json /vitastor/osd/state/1 | jq -r '.kvs[0].mod_revision' +} + +osd1_listens() +{ + ss -Hltnp | grep -q "pid=$OSD1_PID," +} + +# A killed OSD can't revoke its lease, so its state key outlives it. +# The restarted OSD has to wait for that lease to expire instead of exiting, +# and it must not listen until then +old_rev=$(state_rev) +kill -9 $OSD1_PID +start_osd 1 +wait_condition 30 "grep -q 'waiting .* for its lease to expire' ./testdata/osd1.log" "OSD 1 wait" +if osd1_listens; then + format_error "OSD 1 listens while it waits for the old lease" +fi +wait_condition 60 "[[ \$(state_rev) -gt $old_rev ]]" "OSD 1 restart" +new_rev=$(state_rev) +osd1_listens + +# A second OSD with the same number has to give up while the first one is alive +osd1_pid=$OSD1_PID +start_osd_on 1 $(osd_dev dup) +dup_pid=$OSD1_PID +OSD1_PID=$osd1_pid +wait_condition 120 "! kill -0 $dup_pid" "Second OSD 1 exit" +wait $dup_pid && format_error "Second OSD 1 exited with success" +kill -0 $OSD1_PID +if [[ $(state_rev) != $new_rev ]]; then + format_error "Second OSD 1 replaced the state of the first one" +fi + +format_green OK