forked from vitalif/vitastor
Use the same etcd address selection algorithm in the monitor
parent
ce5b6253ab
commit
a8f5c71ae8
|
@ -9,17 +9,18 @@ const options = {};
|
|||
|
||||
for (let i = 2; i < process.argv.length; i++)
|
||||
{
|
||||
if (process.argv[i].substr(0, 2) == '--')
|
||||
if (process.argv[i] === '-h' || process.argv[i] === '--help')
|
||||
{
|
||||
console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' [--verbose 1]'+
|
||||
' [--etcd_address "http://127.0.0.1:2379,..."] [--config_file /etc/vitastor/vitastor.conf]'+
|
||||
' [--etcd_prefix "/vitastor"] [--etcd_start_timeout 5]');
|
||||
process.exit();
|
||||
}
|
||||
else if (process.argv[i].substr(0, 2) == '--')
|
||||
{
|
||||
options[process.argv[i].substr(2)] = process.argv[i+1];
|
||||
i++;
|
||||
}
|
||||
}
|
||||
|
||||
if (!options.etcd_url)
|
||||
{
|
||||
console.error('USAGE: '+process.argv[0]+' '+process.argv[1]+' --etcd_url "http://127.0.0.1:2379,..." --etcd_prefix "/vitastor" --etcd_start_timeout 5 [--verbose 1]');
|
||||
process.exit();
|
||||
}
|
||||
|
||||
new Mon(options).start().catch(e => { console.error(e); process.exit(); });
|
||||
new Mon(options).start().catch(e => { console.error(e); process.exit(1); });
|
||||
|
|
107
mon/mon.js
107
mon/mon.js
|
@ -1,6 +1,7 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const fs = require('fs');
|
||||
const http = require('http');
|
||||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
|
@ -323,17 +324,17 @@ class Mon
|
|||
{
|
||||
constructor(config)
|
||||
{
|
||||
// FIXME: Maybe prefer local etcd
|
||||
this.etcd_urls = [];
|
||||
for (let url of config.etcd_url.split(/,/))
|
||||
this.die = (e) => this._die(e);
|
||||
if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
|
||||
{
|
||||
let scheme = 'http';
|
||||
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
|
||||
if (!/\/[^\/]/.exec(url))
|
||||
url += '/v3';
|
||||
this.etcd_urls.push(scheme+'://'+url);
|
||||
config = {
|
||||
...JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' })),
|
||||
...config,
|
||||
};
|
||||
}
|
||||
this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
|
||||
this.verbose = config.verbose || 0;
|
||||
this.initConfig = config;
|
||||
this.config = {};
|
||||
this.etcd_prefix = config.etcd_prefix || '/vitastor';
|
||||
this.etcd_prefix = this.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1');
|
||||
|
@ -343,6 +344,35 @@ class Mon
|
|||
this.on_stop_cb = () => this.on_stop().catch(console.error);
|
||||
}
|
||||
|
||||
parse_etcd_addresses(addrs)
|
||||
{
|
||||
const is_local_ip = this.local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
|
||||
this.etcd_local = [];
|
||||
this.etcd_urls = [];
|
||||
this.selected_etcd_url = null;
|
||||
this.etcd_urls_to_try = [];
|
||||
if (!(addrs instanceof Array))
|
||||
addrs = addrs ? (''+(addrs||'')).split(/,/) : [];
|
||||
if (!addrs.length)
|
||||
{
|
||||
console.error('Vitastor etcd address(es) not specified. Please set on the command line or in the config file');
|
||||
process.exit(1);
|
||||
}
|
||||
for (let url of addrs)
|
||||
{
|
||||
let scheme = 'http';
|
||||
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
|
||||
const slash = url.indexOf('/');
|
||||
const colon = url.indexOf(':');
|
||||
const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
|
||||
url = scheme+'://'+(slash >= 0 ? url : url+'/v3');
|
||||
if (is_local)
|
||||
this.etcd_local.push(url);
|
||||
else
|
||||
this.etcd_urls.push(url);
|
||||
}
|
||||
}
|
||||
|
||||
async start()
|
||||
{
|
||||
await this.load_config();
|
||||
|
@ -411,6 +441,31 @@ class Mon
|
|||
}
|
||||
}
|
||||
|
||||
pick_next_etcd()
|
||||
{
|
||||
if (this.selected_etcd_url)
|
||||
return this.selected_etcd_url;
|
||||
if (!this.etcd_urls_to_try || !this.etcd_urls_to_try.length)
|
||||
{
|
||||
this.etcd_urls_to_try = [ ...this.etcd_local ];
|
||||
const others = [ ...this.etcd_urls ];
|
||||
while (others.length)
|
||||
{
|
||||
const url = others.splice(0|(others.length*Math.random()), 1);
|
||||
this.etcd_urls_to_try.push(url[0]);
|
||||
}
|
||||
}
|
||||
this.selected_etcd_url = this.etcd_urls_to_try.shift();
|
||||
return this.selected_etcd_url;
|
||||
}
|
||||
|
||||
restart_watcher(cur_addr)
|
||||
{
|
||||
if (this.selected_etcd_url == cur_addr)
|
||||
this.selected_etcd_url = null;
|
||||
this.start_watcher(this.config.etcd_mon_retries).catch(this.die);
|
||||
}
|
||||
|
||||
async start_watcher(retries)
|
||||
{
|
||||
let retry = 0;
|
||||
|
@ -420,7 +475,8 @@ class Mon
|
|||
}
|
||||
while (retries < 0 || retry < retries)
|
||||
{
|
||||
const base = 'ws'+this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)].substr(4);
|
||||
const cur_addr = this.pick_next_etcd();
|
||||
const base = 'ws'+cur_addr.substr(4);
|
||||
const ok = await new Promise((ok, no) =>
|
||||
{
|
||||
const timer_id = setTimeout(() =>
|
||||
|
@ -443,9 +499,9 @@ class Mon
|
|||
});
|
||||
});
|
||||
if (ok)
|
||||
{
|
||||
break;
|
||||
}
|
||||
if (this.selected_etcd_url == cur_addr)
|
||||
this.selected_etcd_url = null;
|
||||
this.ws = null;
|
||||
retry++;
|
||||
}
|
||||
|
@ -453,6 +509,8 @@ class Mon
|
|||
{
|
||||
this.die('Failed to open etcd watch websocket');
|
||||
}
|
||||
const cur_addr = this.selected_etcd_url;
|
||||
this.ws.on('error', () => this.restart_watcher(cur_addr));
|
||||
this.ws.send(JSON.stringify({
|
||||
create_request: {
|
||||
key: b64(this.etcd_prefix+'/'),
|
||||
|
@ -510,7 +568,7 @@ class Mon
|
|||
}
|
||||
if (pg_states_changed)
|
||||
{
|
||||
this.save_last_clean().catch(console.error);
|
||||
this.save_last_clean().catch(this.die);
|
||||
}
|
||||
if (stats_changed)
|
||||
{
|
||||
|
@ -1196,7 +1254,7 @@ class Mon
|
|||
this.recheck_timer = setTimeout(() =>
|
||||
{
|
||||
this.recheck_timer = null;
|
||||
this.recheck_pgs().catch(console.error);
|
||||
this.recheck_pgs().catch(this.die);
|
||||
}, this.config.mon_change_timeout || 1000);
|
||||
}
|
||||
|
||||
|
@ -1463,7 +1521,7 @@ class Mon
|
|||
cur[key_parts[key_parts.length-1]] = kv.value;
|
||||
if (key === 'config/global')
|
||||
{
|
||||
this.config = this.state.config.global;
|
||||
this.config = { ...this.initConfig, ...this.state.config.global };
|
||||
this.check_config();
|
||||
for (const osd_num in this.state.osd.stats)
|
||||
{
|
||||
|
@ -1500,12 +1558,15 @@ class Mon
|
|||
}
|
||||
while (retries < 0 || retry < retries)
|
||||
{
|
||||
const base = this.etcd_urls[Math.floor(Math.random()*this.etcd_urls.length)];
|
||||
retry++;
|
||||
const base = this.pick_next_etcd();
|
||||
const res = await POST(base+path, body, timeout);
|
||||
if (res.error)
|
||||
{
|
||||
console.error('etcd returned error: '+res.error);
|
||||
break;
|
||||
if (this.selected_etcd_url == base)
|
||||
this.selected_etcd_url = null;
|
||||
console.error('failed to query etcd: '+res.error);
|
||||
continue;
|
||||
}
|
||||
if (res.json)
|
||||
{
|
||||
|
@ -1514,26 +1575,20 @@ class Mon
|
|||
console.error('etcd returned error: '+res.json.error);
|
||||
break;
|
||||
}
|
||||
if (this.etcd_urls.length > 1)
|
||||
{
|
||||
// Stick to the same etcd for the rest of calls
|
||||
this.etcd_urls = [ base ];
|
||||
}
|
||||
return res.json;
|
||||
}
|
||||
retry++;
|
||||
}
|
||||
this.die();
|
||||
}
|
||||
|
||||
die(err)
|
||||
_die(err)
|
||||
{
|
||||
// In fact we can just try to rejoin
|
||||
console.error(new Error(err || 'Cluster connection failed'));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
local_ips()
|
||||
local_ips(all)
|
||||
{
|
||||
const ips = [];
|
||||
const ifaces = os.networkInterfaces();
|
||||
|
@ -1541,7 +1596,7 @@ class Mon
|
|||
{
|
||||
for (const iface of ifaces[ifname])
|
||||
{
|
||||
if (iface.family == 'IPv4' && !iface.internal)
|
||||
if (iface.family == 'IPv4' && !iface.internal || all)
|
||||
{
|
||||
ips.push(iface.address);
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue