osd: Wait for the old state lease to expire instead of exiting #140

Open
koolkhel wants to merge 1 commits from koolkhel/vitastor:osd-wait-old-lease into master
5 changed files with 128 additions and 4 deletions

View File

@@ -1301,6 +1301,42 @@ jobs:
echo ""
done
test_osd_restart:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: /root/vitastor/tests/test_osd_restart.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_osd_restart_antietcd:
runs-on: ubuntu-latest
needs: build
container: ${{env.TEST_IMAGE}}:${{github.sha}}
steps:
- name: Run test
id: test
timeout-minutes: 3
run: ANTIETCD=1 /root/vitastor/tests/test_osd_restart.sh
- name: Print logs
if: always() && steps.test.outcome == 'failure'
run: |
for i in /root/vitastor/testdata/*.log /root/vitastor/testdata/*.txt; do
echo "-------- $i --------"
cat $i
echo ""
done
test_switch_primary:
runs-on: ubuntu-latest
needs: build

View File

@@ -186,6 +186,7 @@ class osd_t
osd_messenger_t msgr;
bool use_perms = false;
int etcd_failed_attempts = 0;
uint64_t etcd_lease_ttl = 0;
std::string etcd_lease_id;
json11::Json self_state;
bool loading_peer_config = false;
@@ -308,6 +309,7 @@ class osd_t
json11::Json on_load_pgs_checks_hook();
void on_load_pgs_hook(bool success);
void bind_socket();
void wait_old_osd_state(uint64_t waited = 0);
void acquire_lease();
json11::Json get_osd_state();
void create_osd_state();

View File

@@ -518,8 +518,7 @@ void osd_t::on_load_config_hook(json11::Json::object & global_config)
etcd_global_config_loaded = true;
init_blockstore([this]()
{
bind_socket();
acquire_lease();
wait_old_osd_state();
});
}
@@ -530,14 +529,57 @@ void osd_t::on_reload_config_hook(json11::Json::object & global_config)
renew_lease(true);
}
// A killed OSD leaves its state key in etcd until its lease expires, and etcd restarts lease
// TTLs when its leader changes. So recheck the key for up to 3 TTLs, and only then start listening
void osd_t::wait_old_osd_state(uint64_t waited)
{
std::string state_key = st_cli->etcd_prefix+"/osd/state/"+std::to_string(osd_num);
// Maximum lease TTL is (report interval) + retries * (timeout + repeat interval)
etcd_lease_ttl = etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000;
st_cli->etcd_txn(json11::Json::object {
{ "success", json11::Json::array {
json11::Json::object {
{ "request_range", json11::Json::object {
{ "key", base64_encode(state_key) },
} }
},
} },
}, st_cli->etcd_quick_timeout, 0, 0, [this, state_key, waited](std::string err, json11::Json data)
{
uint64_t max_wait = 3000*etcd_lease_ttl;
if (err != "")
{
printf("Error reading OSD state from etcd: %s, retrying\n", err.c_str());
}
else if (data["responses"][0]["response_range"]["kvs"][0]["lease"].is_null() || waited >= max_wait)
{
// If the key is still there, create_osd_state() says that the OSD is up and exits
bind_socket();
acquire_lease();
return;
}
else if (!waited)
{
printf(
"[OSD %ju] Key %s already exists in etcd, waiting up to %ju seconds for its lease to expire\n",
this->osd_num, state_key.c_str(), max_wait/1000
);
}
uint64_t next = err != "" ? waited : waited+st_cli->etcd_quick_timeout;

Может быть не пытаться спать полный интервал при неудаче, а просто ретраить получение лизы до успеха с равными интервалами типа 1 секунды или более?

Может быть не пытаться спать полный интервал при неудаче, а просто ретраить получение лизы до успеха с равными интервалами типа 1 секунды или более?

Сделал так: убрал /lease/timetolive и расчёт интервала совсем. Теперь OSD просто перепроверяет ключ раз в etcd_quick_timeout (1 с по умолчанию), пока он не исчезнет, одинаково для etcd и antietcd. Ограничение в 3 TTL оставил: после него create_osd_state() находит ключ и выходит с «OSD N is still up», как в master.

Сделал так: убрал `/lease/timetolive` и расчёт интервала совсем. Теперь OSD просто перепроверяет ключ раз в `etcd_quick_timeout` (1 с по умолчанию), пока он не исчезнет, одинаково для etcd и antietcd. Ограничение в 3 TTL оставил: после него `create_osd_state()` находит ключ и выходит с «OSD N is still up», как в master.
tfd->set_timer(st_cli->etcd_quick_timeout, false, [this, next](int timer_id)
{
wait_old_osd_state(next);
});
});
}
// Acquire lease
void osd_t::acquire_lease()
{
// Apply no_inode_stats before the first statistics report
apply_no_inode_stats();
// Maximum lease TTL is (report interval) + retries * (timeout + repeat interval)
st_cli->etcd_call("/lease/grant", json11::Json::object {
{ "TTL", etcd_report_interval+(st_cli->max_etcd_attempts*(2*st_cli->etcd_quick_timeout)+999)/1000 }
{ "TTL", etcd_lease_ttl }
}, st_cli->etcd_quick_timeout, 0, 0, [this](std::string err, json11::Json data)
{
if (err != "" || data["ID"].string_value() == "")

View File

@@ -106,6 +106,9 @@ OLD=1 SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh
./test_root_node.sh
./test_osd_restart.sh
ANTIETCD=1 ./test_osd_restart.sh
./test_switch_primary.sh
./test_write.sh

41
tests/test_osd_restart.sh Executable file
View File

@@ -0,0 +1,41 @@
#!/bin/bash -ex
. `dirname $0`/run_3osds.sh
state_rev()
{
$ETCDCTL get --write-out=json /vitastor/osd/state/1 | jq -r '.kvs[0].mod_revision'
}
osd1_listens()
{
ss -Hltnp | grep -q "pid=$OSD1_PID,"
}
# A killed OSD can't revoke its lease, so its state key outlives it.
# The restarted OSD has to wait for that lease to expire instead of exiting,
# and it must not listen until then
old_rev=$(state_rev)
kill -9 $OSD1_PID
start_osd 1
wait_condition 30 "grep -q 'waiting .* for its lease to expire' ./testdata/osd1.log" "OSD 1 wait"
if osd1_listens; then
format_error "OSD 1 listens while it waits for the old lease"
fi
wait_condition 60 "[[ \$(state_rev) -gt $old_rev ]]" "OSD 1 restart"
new_rev=$(state_rev)
osd1_listens
# A second OSD with the same number has to give up while the first one is alive
osd1_pid=$OSD1_PID
start_osd_on 1 $(osd_dev dup)
dup_pid=$OSD1_PID
OSD1_PID=$osd1_pid
wait_condition 120 "! kill -0 $dup_pid" "Second OSD 1 exit"
wait $dup_pid && format_error "Second OSD 1 exited with success"
kill -0 $OSD1_PID
if [[ $(state_rev) != $new_rev ]]; then
format_error "Second OSD 1 replaced the state of the first one"
fi
format_green OK