osd: Wait for the old state lease to expire instead of exiting #140
@@ -1301,6 +1301,42 @@ jobs:
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_osd_restart:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: /root/vitastor/tests/test_osd_restart.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_osd_restart_antietcd:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
container: ${{env.TEST_IMAGE}}:${{github.sha}}
|
||||
steps:
|
||||
- name: Run test
|
||||
id: test
|
||||
timeout-minutes: 3
|
||||
run: ANTIETCD=1 /root/vitastor/tests/test_osd_restart.sh
|
||||
- name: Print logs
|
||||
if: always() && steps.test.outcome == 'failure'
|
||||
run: |
|
||||
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
|
||||
echo "-------- $i --------"
|
||||
cat $i
|
||||
echo ""
|
||||
done
|
||||
|
||||
test_switch_primary:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build
|
||||
|
||||
@@ -186,6 +186,7 @@ class osd_t
|
||||
osd_messenger_t msgr;
|
||||
bool use_perms = false;
|
||||
int etcd_failed_attempts = 0;
|
||||
uint64_t etcd_lease_ttl = 0;
|
||||
std::string etcd_lease_id;
|
||||
json11::Json self_state;
|
||||
bool loading_peer_config = false;
|
||||
@@ -308,6 +309,7 @@ class osd_t
|
||||
json11::Json on_load_pgs_checks_hook();
|
||||
void on_load_pgs_hook(bool success);
|
||||
void bind_socket();
|
||||
void wait_old_osd_state(uint64_t waited = 0);
|
||||
void acquire_lease();
|
||||
json11::Json get_osd_state();
|
||||
void create_osd_state();
|
||||
|
||||
@@ -518,8 +518,7 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config)
|
||||
etcd_global_config_loaded = true;
|
||||
init_blockstore([this]()
|
||||
{
|
||||
bind_socket();
|
||||
acquire_lease();
|
||||
wait_old_osd_state();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -530,14 +529,57 @@ void osd_t::on_reload_config_hook(json11::Json::object & global_config)
|
||||
renew_lease(true);
|
||||
}
|
||||
|
||||
// A killed OSD leaves its state key in etcd until its lease expires, and etcd restarts lease
|
||||
// TTLs when its leader changes. So recheck the key for up to 3 TTLs, and only then start listening
|
||||
void osd_t::wait_old_osd_state(uint64_t waited)
|
||||
{
|
||||
std::string state_key = st_cli->etcd_prefix+"/osd/state/"+std::to_string(osd_num);
|
||||
// Maximum lease TTL is (report interval) + retries * (timeout + repeat interval)
|
||||
etcd_lease_ttl = etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000;
|
||||
st_cli->etcd_txn(json11::Json::object {
|
||||
{ "success", json11::Json::array {
|
||||
json11::Json::object {
|
||||
{ "request_range", json11::Json::object {
|
||||
{ "key", base64_encode(state_key) },
|
||||
} }
|
||||
},
|
||||
} },
|
||||
}, st_cli->etcd_quick_timeout, 0, 0, [this, state_key, waited](std::string err, json11::Json data)
|
||||
{
|
||||
uint64_t max_wait = 3000*etcd_lease_ttl;
|
||||
if (err != "")
|
||||
{
|
||||
printf("Error reading OSD state from etcd: %s, retrying\n", err.c_str());
|
||||
}
|
||||
else if (data["responses"][0]["response_range"]["kvs"][0]["lease"].is_null() || waited >= max_wait)
|
||||
{
|
||||
// If the key is still there, create_osd_state() says that the OSD is up and exits
|
||||
bind_socket();
|
||||
acquire_lease();
|
||||
return;
|
||||
}
|
||||
else if (!waited)
|
||||
{
|
||||
printf(
|
||||
"[OSD %ju] Key %s already exists in etcd, waiting up to %ju seconds for its lease to expire\n",
|
||||
this->osd_num, state_key.c_str(), max_wait/1000
|
||||
);
|
||||
}
|
||||
uint64_t next = err != "" ? waited : waited+st_cli->etcd_quick_timeout;
|
||||
|
|
||||
tfd->set_timer(st_cli->etcd_quick_timeout, false, [this, next](int timer_id)
|
||||
{
|
||||
wait_old_osd_state(next);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Acquire lease
|
||||
void osd_t::acquire_lease()
|
||||
{
|
||||
// Apply no_inode_stats before the first statistics report
|
||||
apply_no_inode_stats();
|
||||
// Maximum lease TTL is (report interval) + retries * (timeout + repeat interval)
|
||||
st_cli->etcd_call("/lease/grant", json11::Json::object {
|
||||
{ "TTL", etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000 }
|
||||
{ "TTL", etcd_lease_ttl }
|
||||
}, st_cli->etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
|
||||
{
|
||||
if (err != "" || data["ID"].string_value() == "")
|
||||
|
||||
@@ -106,6 +106,9 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
|
||||
|
||||
./test_root_node.sh
|
||||
|
||||
./test_osd_restart.sh
|
||||
ANTIETCD=1 ./test_osd_restart.sh
|
||||
|
||||
./test_switch_primary.sh
|
||||
|
||||
./test_write.sh
|
||||
|
||||
41
tests/test_osd_restart.sh
Executable file
41
tests/test_osd_restart.sh
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash -ex
|
||||
|
||||
. `dirname $0`/run_3osds.sh
|
||||
|
||||
state_rev()
|
||||
{
|
||||
$ETCDCTL get --write-out=json /vitastor/osd/state/1 | jq -r '.kvs[0].mod_revision'
|
||||
}
|
||||
|
||||
osd1_listens()
|
||||
{
|
||||
ss -Hltnp | grep -q "pid=$OSD1_PID,"
|
||||
}
|
||||
|
||||
# A killed OSD can't revoke its lease, so its state key outlives it.
|
||||
# The restarted OSD has to wait for that lease to expire instead of exiting,
|
||||
# and it must not listen until then
|
||||
old_rev=$(state_rev)
|
||||
kill -9 $OSD1_PID
|
||||
start_osd 1
|
||||
wait_condition 30 "grep -q 'waiting .* for its lease to expire' ./testdata/osd1.log" "OSD 1 wait"
|
||||
if osd1_listens; then
|
||||
format_error "OSD 1 listens while it waits for the old lease"
|
||||
fi
|
||||
wait_condition 60 "[[ \$(state_rev) -gt $old_rev ]]" "OSD 1 restart"
|
||||
new_rev=$(state_rev)
|
||||
osd1_listens
|
||||
|
||||
# A second OSD with the same number has to give up while the first one is alive
|
||||
osd1_pid=$OSD1_PID
|
||||
start_osd_on 1 $(osd_dev dup)
|
||||
dup_pid=$OSD1_PID
|
||||
OSD1_PID=$osd1_pid
|
||||
wait_condition 120 "! kill -0 $dup_pid" "Second OSD 1 exit"
|
||||
wait $dup_pid && format_error "Second OSD 1 exited with success"
|
||||
kill -0 $OSD1_PID
|
||||
if [[ $(state_rev) != $new_rev ]]; then
|
||||
format_error "Second OSD 1 replaced the state of the first one"
|
||||
fi
|
||||
|
||||
format_green OK
|
||||
Reference in New Issue
Block a user
Может быть не пытаться спать полный интервал при неудаче, а просто ретраить получение лизы до успеха с равными интервалами типа 1 секунды или более?
Сделал так: убрал
/lease/timetoliveи расчёт интервала совсем. Теперь OSD просто перепроверяет ключ раз вetcd_quick_timeout(1 с по умолчанию), пока он не исчезнет, одинаково для etcd и antietcd. Ограничение в 3 TTL оставил: после негоcreate_osd_state()находит ключ и выходит с «OSD N is still up», как в master.