vitalif
/
vitastor
5
4
Fork 9
Code Issues 1 Pull Requests 2 Actions Releases 69 Wiki Activity

Add self-restart support to monitor (mainly for tests)

mon-self-restart
Vitaliy Filippov 2023-05-08 20:38:56 +03:00
parent a409598b16
commit 77155ab7bd
2 changed files with 35 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
mon/mon-main.js
View File

@ -13,7 +13,7 @@ for (let i = 2; i < process.argv.length; i++)
{ {
console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+ console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+
' [--etcd_address "http://127.0.0.1:2379,..."] [--config_path /etc/vitastor/vitastor.conf]'+ ' [--etcd_address "http://127.0.0.1:2379,..."] [--config_path /etc/vitastor/vitastor.conf]'+
' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5]'); ' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5] [--restart_interval 5]');
process.exit(); process.exit();
} }
else if (process.argv[i].substr(0, 2) == '--') else if (process.argv[i].substr(0, 2) == '--')

43
mon/mon.js
View File

@ -561,7 +561,7 @@ class Mon
} }
if (!this.ws) if (!this.ws)
{ {
this.die('Failed to open etcd watch websocket'); await this.die('Failed to open etcd watch websocket');
} }
const cur_addr = this.selected_etcd_url; const cur_addr = this.selected_etcd_url;
this.ws_alive = true; this.ws_alive = true;
@ -728,7 +728,7 @@ class Mon
const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries); const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
if (!res.result.TTL) if (!res.result.TTL)
{ {
this.die('Lease expired'); await this.die('Lease expired');
} }
}, this.config.etcd_mon_timeout); }, this.config.etcd_mon_timeout);
if (!this.signals_set) if (!this.signals_set)
@ -741,9 +741,32 @@ class Mon
async on_stop(status) async on_stop(status)
{ {
clearInterval(this.lease_timer); if (this.ws_keepalive_timer)
await this.etcd_call('/lease/revoke', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries); {
process.exit(status); clearInterval(this.ws_keepalive_timer);
this.ws_keepalive_timer = null;
}
if (this.lease_timer)
{
clearInterval(this.lease_timer);
this.lease_timer = null;
}
if (this.etcd_lease_id)
{
const lease_id = this.etcd_lease_id;
this.etcd_lease_id = null;
await this.etcd_call('/lease/revoke', { ID: lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
}
if (!status || !this.initConfig.restart_interval)
{
process.exit(status);
}
else
{
console.log('Restarting after '+this.initConfig.restart_interval+' seconds');
await new Promise(ok => setTimeout(ok, this.initConfig.restart_interval*1000));
await this.start();
}
} }
async become_master() async become_master()
@ -1771,14 +1794,13 @@ class Mon
return res.json; return res.json;
} }
} }
this.die(); await this.die();
} }
_die(err) async _die(err)
{ {
// In fact we can just try to rejoin
console.error(new Error(err || 'Cluster connection failed')); console.error(new Error(err || 'Cluster connection failed'));
process.exit(1); await this.on_stop(1);
} }
local_ips(all) local_ips(all)
@ -1823,6 +1845,7 @@ function POST(url, body, timeout)
clearTimeout(timer_id); clearTimeout(timer_id);
let res_body = ''; let res_body = '';
res.setEncoding('utf8'); res.setEncoding('utf8');
res.on('error', no);
res.on('data', chunk => { res_body += chunk; }); res.on('data', chunk => { res_body += chunk; });
res.on('end', () => res.on('end', () =>
{ {
@ -1842,6 +1865,8 @@ function POST(url, body, timeout)
} }
}); });
}); });
req.on('error', no);
req.on('close', () => no(new Error('Connection closed prematurely')));
req.write(body_text); req.write(body_text);
req.end(); req.end();
}); });