Compare commits
No commits in common. "df668286fbdc92043b17c6e1da2a116e58bf1bc3" and "79f1d1969b256a939037df6f4ba1963370d82e16" have entirely different histories.
df668286fb
...
79f1d1969b
|
@ -8,13 +8,6 @@
|
|||
|
||||
These parameters only apply to Monitors.
|
||||
|
||||
- [enable_prometheus](#enable_prometheus)
|
||||
- [mon_http_port](#mon_http_port)
|
||||
- [mon_http_ip](#mon_http_ip)
|
||||
- [mon_https_cert](#mon_https_cert)
|
||||
- [mon_https_key](#mon_https_key)
|
||||
- [mon_https_client_auth](#mon_https_client_auth)
|
||||
- [mon_https_ca](#mon_https_ca)
|
||||
- [etcd_mon_ttl](#etcd_mon_ttl)
|
||||
- [etcd_mon_timeout](#etcd_mon_timeout)
|
||||
- [etcd_mon_retries](#etcd_mon_retries)
|
||||
|
@ -24,51 +17,6 @@ These parameters only apply to Monitors.
|
|||
- [placement_levels](#placement_levels)
|
||||
- [use_old_pg_combinator](#use_old_pg_combinator)
|
||||
|
||||
## enable_prometheus
|
||||
|
||||
- Type: boolean
|
||||
- Default: true
|
||||
|
||||
Enable built-in Prometheus metrics exporter
|
||||
|
||||
## mon_http_port
|
||||
|
||||
- Type: integer
|
||||
- Default: 8060
|
||||
|
||||
HTTP port for monitors to listen on (including metrics exporter)
|
||||
|
||||
## mon_http_ip
|
||||
|
||||
- Type: string
|
||||
|
||||
IP address for monitors to listen on (all addresses by default)
|
||||
|
||||
## mon_https_cert
|
||||
|
||||
- Type: string
|
||||
|
||||
Path to PEM SSL certificate file for monitor to listen using HTTPS
|
||||
|
||||
## mon_https_key
|
||||
|
||||
- Type: string
|
||||
|
||||
Path to PEM SSL private key file for monitor to listen using HTTPS
|
||||
|
||||
## mon_https_client_auth
|
||||
|
||||
- Type: boolean
|
||||
- Default: false
|
||||
|
||||
Enable HTTPS client certificate-based authorization for monitor connections
|
||||
|
||||
## mon_https_ca
|
||||
|
||||
- Type: string
|
||||
|
||||
Path to CA certificate for client HTTPS authorization
|
||||
|
||||
## etcd_mon_ttl
|
||||
|
||||
- Type: seconds
|
||||
|
|
|
@ -8,13 +8,6 @@
|
|||
|
||||
Данные параметры используются только мониторами Vitastor.
|
||||
|
||||
- [enable_prometheus](#enable_prometheus)
|
||||
- [mon_http_port](#mon_http_port)
|
||||
- [mon_http_ip](#mon_http_ip)
|
||||
- [mon_https_cert](#mon_https_cert)
|
||||
- [mon_https_key](#mon_https_key)
|
||||
- [mon_https_client_auth](#mon_https_client_auth)
|
||||
- [mon_https_ca](#mon_https_ca)
|
||||
- [etcd_mon_ttl](#etcd_mon_ttl)
|
||||
- [etcd_mon_timeout](#etcd_mon_timeout)
|
||||
- [etcd_mon_retries](#etcd_mon_retries)
|
||||
|
@ -24,51 +17,6 @@
|
|||
- [placement_levels](#placement_levels)
|
||||
- [use_old_pg_combinator](#use_old_pg_combinator)
|
||||
|
||||
## enable_prometheus
|
||||
|
||||
- Тип: булево (да/нет)
|
||||
- Значение по умолчанию: true
|
||||
|
||||
Включить встроенный Prometheus-экспортер метри
|
||||
|
||||
## mon_http_port
|
||||
|
||||
- Тип: целое число
|
||||
- Значение по умолчанию: 8060
|
||||
|
||||
Порт, на котором мониторы принимают HTTP-соединения (в том числе для отдачи метрик)
|
||||
|
||||
## mon_http_ip
|
||||
|
||||
- Тип: строка
|
||||
|
||||
IP-адрес, на котором мониторы принимают HTTP-соединения (по умолчанию все адреса)
|
||||
|
||||
## mon_https_cert
|
||||
|
||||
- Тип: строка
|
||||
|
||||
Путь к PEM-файлу SSL-сертификата для монитора, чтобы принимать соединения через HTTPS
|
||||
|
||||
## mon_https_key
|
||||
|
||||
- Тип: строка
|
||||
|
||||
Путь к PEM-файлу секретного SSL-ключа для монитора, чтобы принимать соединения через HTTPS
|
||||
|
||||
## mon_https_client_auth
|
||||
|
||||
- Тип: булево (да/нет)
|
||||
- Значение по умолчанию: false
|
||||
|
||||
Включить в HTTPS-сервере монитора авторизацию по клиентским сертификатам
|
||||
|
||||
## mon_https_ca
|
||||
|
||||
- Тип: строка
|
||||
|
||||
Путь к удостоверяющему сертификату для авторизации клиентских HTTPS соединений
|
||||
|
||||
## etcd_mon_ttl
|
||||
|
||||
- Тип: секунды
|
||||
|
|
|
@ -1,34 +1,3 @@
|
|||
- name: enable_prometheus
|
||||
type: bool
|
||||
default: true
|
||||
info: Enable built-in Prometheus metrics exporter
|
||||
info_ru: Включить встроенный Prometheus-экспортер метри
|
||||
- name: mon_http_port
|
||||
type: int
|
||||
default: 8060
|
||||
info: HTTP port for monitors to listen on (including metrics exporter)
|
||||
info_ru: Порт, на котором мониторы принимают HTTP-соединения (в том числе для отдачи метрик)
|
||||
- name: mon_http_ip
|
||||
type: string
|
||||
info: IP address for monitors to listen on (all addresses by default)
|
||||
info_ru: IP-адрес, на котором мониторы принимают HTTP-соединения (по умолчанию все адреса)
|
||||
- name: mon_https_cert
|
||||
type: string
|
||||
info: Path to PEM SSL certificate file for monitor to listen using HTTPS
|
||||
info_ru: Путь к PEM-файлу SSL-сертификата для монитора, чтобы принимать соединения через HTTPS
|
||||
- name: mon_https_key
|
||||
type: string
|
||||
info: Path to PEM SSL private key file for monitor to listen using HTTPS
|
||||
info_ru: Путь к PEM-файлу секретного SSL-ключа для монитора, чтобы принимать соединения через HTTPS
|
||||
- name: mon_https_client_auth
|
||||
type: bool
|
||||
default: false
|
||||
info: Enable HTTPS client certificate-based authorization for monitor connections
|
||||
info_ru: Включить в HTTPS-сервере монитора авторизацию по клиентским сертификатам
|
||||
- name: mon_https_ca
|
||||
type: string
|
||||
info: Path to CA certificate for client HTTPS authorization
|
||||
info_ru: Путь к удостоверяющему сертификату для авторизации клиентских HTTPS соединений
|
||||
- name: etcd_mon_ttl
|
||||
type: sec
|
||||
min: 5
|
||||
|
|
|
@ -245,9 +245,6 @@ const etcd_tree = {
|
|||
stats: {
|
||||
/* <osd_num_t>: {
|
||||
time: number, // unix time
|
||||
data_block_size: uint64_t, // bytes
|
||||
bitmap_granularity: uint64_t, // bytes
|
||||
immediate_commit: "all"|"small"|"none",
|
||||
blockstore_ready: boolean,
|
||||
size: uint64_t, // bytes
|
||||
free: uint64_t, // bytes
|
||||
|
@ -285,7 +282,7 @@ const etcd_tree = {
|
|||
master: {
|
||||
/* ip: [ string ], id: uint64_t */
|
||||
},
|
||||
member: {
|
||||
standby: {
|
||||
/* <uint64_t>: { ip: [ string ] }, */
|
||||
},
|
||||
},
|
||||
|
|
|
@ -1,46 +0,0 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const fsp = require('fs').promises;
|
||||
const http = require('http');
|
||||
const https = require('https');
|
||||
|
||||
async function create_http_server(cfg, handler)
|
||||
{
|
||||
let server;
|
||||
if (cfg.mon_https_cert)
|
||||
{
|
||||
const tls = {
|
||||
key: await fsp.readFile(cfg.mon_https_key),
|
||||
cert: await fsp.readFile(cfg.mon_https_cert),
|
||||
};
|
||||
if (cfg.mon_https_ca)
|
||||
{
|
||||
tls.mon_https_ca = await fsp.readFile(cfg.mon_https_ca);
|
||||
}
|
||||
if (cfg.mon_https_client_auth)
|
||||
{
|
||||
tls.requestCert = true;
|
||||
}
|
||||
server = https.createServer(tls, handler);
|
||||
}
|
||||
else
|
||||
{
|
||||
server = http.createServer(handler);
|
||||
}
|
||||
try
|
||||
{
|
||||
server.listen(cfg.mon_http_port || 8060, cfg.mon_http_ip || undefined);
|
||||
}
|
||||
catch (e)
|
||||
{
|
||||
console.error(
|
||||
'HTTP server disabled because listen at address: '+
|
||||
(cfg.mon_http_ip || '')+':'+(cfg.mon_http_port || 9090)+' failed with error: '+e
|
||||
);
|
||||
return null;
|
||||
}
|
||||
return server;
|
||||
}
|
||||
|
||||
module.exports = { create_http_server };
|
30
mon/mon.js
30
mon/mon.js
|
@ -1,13 +1,10 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const { URL } = require('url');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
const os = require('os');
|
||||
const EtcdAdapter = require('./etcd_adapter.js');
|
||||
const { create_http_server } = require('./http_server.js');
|
||||
const { export_prometheus_metrics } = require('./prometheus.js');
|
||||
const { etcd_tree, etcd_allow, etcd_nonempty_keys } = require('./etcd_schema.js');
|
||||
const { validate_pool_cfg } = require('./pool_config.js');
|
||||
const { sum_op_stats, sum_object_counts, sum_inode_stats, serialize_bigints } = require('./stats.js');
|
||||
|
@ -63,32 +60,6 @@ class Mon
|
|||
this.recheck_pgs_active = false;
|
||||
this.etcd = new EtcdAdapter(this);
|
||||
this.etcd.parse_config(this.config);
|
||||
this.watcher_active = false;
|
||||
if (this.config.enable_prometheus || !('enable_prometheus' in this.config))
|
||||
{
|
||||
this.http = create_http_server(this.config, (req, res) =>
|
||||
{
|
||||
const u = new URL(req.url, 'http://'+(req.headers.host || 'localhost'));
|
||||
if (u.pathname.replace(/\/+$/, '') == (this.config.prometheus_path||'/metrics'))
|
||||
{
|
||||
if (!this.watcher_active)
|
||||
{
|
||||
res.writeHead(503);
|
||||
res.write('Monitor is in standby mode. Please retrieve metrics from master monitor instance\n');
|
||||
}
|
||||
else
|
||||
{
|
||||
res.write(export_prometheus_metrics(this.state));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
res.writeHead(404);
|
||||
res.write('Not found. Metrics path: '+(this.config.prometheus_path||'/metrics\n'));
|
||||
}
|
||||
res.end();
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async start()
|
||||
|
@ -98,7 +69,6 @@ class Mon
|
|||
await this.etcd.become_master();
|
||||
await this.load_cluster_state();
|
||||
await this.etcd.start_watcher(this.config.etcd_mon_retries);
|
||||
this.watcher_active = true;
|
||||
for (const pool_id in this.state.config.pools)
|
||||
{
|
||||
if (!this.state.pool.stats[pool_id] ||
|
||||
|
|
|
@ -1,220 +0,0 @@
|
|||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 (see README.md for details)
|
||||
|
||||
const metric_help =
|
||||
`# HELP vitastor_object_bytes Total size of objects in cluster in bytes
|
||||
# TYPE vitastor_object_bytes gauge
|
||||
# HELP vitastor_object_count Total number of objects in cluster
|
||||
# TYPE vitastor_object_count gauge
|
||||
# HELP vitastor_stat_count Total operation count
|
||||
# TYPE vitastor_stat_count counter
|
||||
# HELP vitastor_stat_usec Total operation latency in usec
|
||||
# TYPE vitastor_stat_usec counter
|
||||
# HELP vitastor_stat_bytes Total operation size in bytes
|
||||
# HELP vitastor_stat_bytes counter
|
||||
|
||||
# HELP vitastor_image_raw_used Image raw used size in bytes
|
||||
# TYPE vitastor_image_raw_used counter
|
||||
# HELP vitastor_image_stat_count Per-image total operation count
|
||||
# TYPE vitastor_image_stat_count counter
|
||||
# HELP vitastor_image_stat_usec Per-image total operation latency
|
||||
# TYPE vitastor_image_stat_usec counter
|
||||
# HELP vitastor_image_stat_bytes Per-image total operation size in bytes
|
||||
# TYPE vitastor_image_stat_bytes counter
|
||||
|
||||
# HELP vitastor_osd_status OSD up/down status
|
||||
# TYPE vitastor_osd_status gauge
|
||||
# HELP vitastor_osd_size_bytes OSD total space in bytes
|
||||
# TYPE vitastor_osd_size_bytes gauge
|
||||
# HELP vitastor_osd_free_bytes OSD free space in bytes
|
||||
# TYPE vitastor_osd_free_bytes gauge
|
||||
# HELP vitastor_osd_stat_count Per-image total operation count
|
||||
# TYPE vitastor_osd_stat_count counter
|
||||
# HELP vitastor_osd_stat_usec Per-image total operation latency
|
||||
# TYPE vitastor_osd_stat_usec counter
|
||||
# HELP vitastor_osd_stat_bytes Per-image total operation size in bytes
|
||||
# TYPE vitastor_osd_stat_bytes counter
|
||||
|
||||
# HELP vitastor_monitor_info Monitor info, 1 is master, 0 is standby
|
||||
# TYPE vitastor_monitor_info gauge
|
||||
|
||||
# HELP vitastor_pool_info Pool configuration (in labels)
|
||||
# TYPE vitastor_pool_info gauge
|
||||
# HELP vitastor_pool_status Pool up/down status
|
||||
# TYPE vitastor_pool_status gauge
|
||||
# HELP vitastor_pool_raw_to_usable Raw to usable space ratio
|
||||
# TYPE vitastor_pool_raw_to_usable gauge
|
||||
# HELP vitastor_pool_space_efficiency Pool space usage efficiency
|
||||
# TYPE vitastor_pool_space_efficiency gauge
|
||||
# HELP vitastor_pool_total_raw_tb Total raw space in pool in TB
|
||||
# TYPE vitastor_pool_total_raw_tb gauge
|
||||
# HELP vitastor_pool_used_raw_tb Used raw space in pool in TB
|
||||
# TYPE vitastor_pool_used_raw_tb gauge
|
||||
# HELP vitastor_pg_count PG counts by state
|
||||
# HELP vitastor_pg_count gauge
|
||||
|
||||
`;
|
||||
|
||||
function export_prometheus_metrics(st)
|
||||
{
|
||||
let res = metric_help;
|
||||
|
||||
// Global statistics
|
||||
|
||||
for (const k in st.stats.object_bytes)
|
||||
{
|
||||
res += `vitastor_object_bytes{object_type="${k}"} ${st.stats.object_bytes[k]}\n`;
|
||||
}
|
||||
|
||||
for (const k in st.stats.object_counts)
|
||||
{
|
||||
res += `vitastor_object_count{object_type="${k}"} ${st.stats.object_counts[k]}\n`;
|
||||
}
|
||||
|
||||
for (const typ of [ 'op', 'subop', 'recovery' ])
|
||||
{
|
||||
for (const op in st.stats[typ+"_stats"]||{})
|
||||
{
|
||||
const op_stat = st.stats[typ+"_stats"][op];
|
||||
for (const key of [ 'count', 'usec', 'bytes' ])
|
||||
{
|
||||
res += `vitastor_stat_${key}{op="${op}",op_type="${typ}"} ${op_stat[key]||0}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Per-image statistics
|
||||
|
||||
for (const pool in st.inode.stats)
|
||||
{
|
||||
for (const inode in st.inode.stats[pool])
|
||||
{
|
||||
const ist = st.inode.stats[pool][inode];
|
||||
const inode_name = ((st.config.inode[pool]||{})[inode]||{}).name||'';
|
||||
const inode_label = `image_name="${addslashes(inode_name)}",inode_num="${inode}",pool_id="${pool}"`;
|
||||
res += `vitastor_image_raw_used{${inode_label}} ${ist.raw_used||0}\n`;
|
||||
for (const op of [ 'read', 'write', 'delete' ])
|
||||
{
|
||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
||||
{
|
||||
if (ist[op])
|
||||
{
|
||||
res += `vitastor_image_stat_${k}{${inode_label},op="${op}"} ${ist[op][k]||0}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Per-OSD statistics
|
||||
|
||||
for (const osd in st.osd.stats)
|
||||
{
|
||||
const osd_stat = st.osd.stats[osd];
|
||||
const up = st.osd.state[osd] && st.osd.state[osd].state == 'up' ? 1 : 0;
|
||||
res += `vitastor_osd_status{host="${addslashes(osd_stat.host)}",osd_num="${osd}"} ${up}\n`;
|
||||
res += `vitastor_osd_size_bytes{osd_num="${osd}"} ${osd_stat.size||0}\n`;
|
||||
res += `vitastor_osd_free_bytes{osd_num="${osd}"} ${osd_stat.free||0}\n`;
|
||||
for (const op in osd_stat.op_stats)
|
||||
{
|
||||
const ist = osd_stat.op_stats[op];
|
||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
||||
{
|
||||
res += `vitastor_osd_stat_${k}{osd_num="${osd}",op="${op}",op_type="op"} ${ist[k]||0}\n`;
|
||||
}
|
||||
}
|
||||
for (const op in osd_stat.subop_stats)
|
||||
{
|
||||
const ist = osd_stat.subop_stats[op];
|
||||
for (const k of [ 'count', 'usec', 'bytes' ])
|
||||
{
|
||||
res += `vitastor_osd_stat_${k}{osd_num="${osd}",op="${op}",op_type="subop"} ${ist[k]||0}\n`;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Monitor statistics
|
||||
|
||||
for (const mon_id in st.mon.member)
|
||||
{
|
||||
const mon = st.mon.member[mon_id];
|
||||
const master = st.mon.master && st.mon.master.id == mon_id ? 1 : 0;
|
||||
const ip = (mon.ip instanceof Array ? mon.ip[0] : mon.ip) || '';
|
||||
res += `vitastor_monitor_info{monitor_hostname="${addslashes(mon.hostname)}",monitor_id="${mon_id}",monitor_ip="${addslashes(ip)}"} ${master}\n`;
|
||||
}
|
||||
|
||||
// Per-pool statistics
|
||||
|
||||
for (const pool_id in st.config.pools)
|
||||
{
|
||||
const pool_cfg = st.config.pools[pool_id];
|
||||
const pool_label = `pool_id="${pool_id}",pool_name="${addslashes(pool_cfg.name)}"`;
|
||||
const pool_stat = st.pool.stats[pool_id];
|
||||
res += `vitastor_pool_info{${pool_label}`+
|
||||
`,pool_scheme="${addslashes(pool_cfg.scheme)}"`+
|
||||
`,pg_size="${pool_cfg.pg_size||0}",pg_minsize="${pool_cfg.pg_minsize||0}"`+
|
||||
`,parity_chunks="${pool_cfg.parity_chunks||0}",pg_count="${pool_cfg.pg_count||0}"`+
|
||||
`,failure_domain="${addslashes(pool_cfg.failure_domain)}"`+
|
||||
`} 1\n`;
|
||||
if (!pool_stat)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
res += `vitastor_pool_raw_to_usable{${pool_label}} ${pool_stat.raw_to_usable||0}\n`;
|
||||
res += `vitastor_pool_space_efficiency{${pool_label}} ${pool_stat.space_efficiency||0}\n`;
|
||||
res += `vitastor_pool_total_raw_tb{${pool_label}} ${pool_stat.total_raw_tb||0}\n`;
|
||||
res += `vitastor_pool_used_raw_tb{${pool_label}} ${pool_stat.used_raw_tb||0}\n`;
|
||||
|
||||
// PG states and pool up/down status
|
||||
const real_pg_count = (Object.keys(((st.config.pgs||{}).items||{})[pool_id]||{}).length) || (0|pool_cfg.pg_count);
|
||||
const per_state = {
|
||||
active: 0,
|
||||
starting: 0,
|
||||
peering: 0,
|
||||
incomplete: 0,
|
||||
repeering: 0,
|
||||
stopping: 0,
|
||||
offline: 0,
|
||||
degraded: 0,
|
||||
has_inconsistent: 0,
|
||||
has_corrupted: 0,
|
||||
has_incomplete: 0,
|
||||
has_degraded: 0,
|
||||
has_misplaced: 0,
|
||||
has_unclean: 0,
|
||||
has_invalid: 0,
|
||||
left_on_dead: 0,
|
||||
scrubbing: 0,
|
||||
};
|
||||
const pool_pg_states = st.pg.state[pool_id] || {};
|
||||
for (let i = 1; i <= real_pg_count; i++)
|
||||
{
|
||||
if (!pool_pg_states[i])
|
||||
{
|
||||
per_state['offline'] = 1 + (per_state['offline']|0);
|
||||
}
|
||||
else
|
||||
{
|
||||
for (const st_name of pool_pg_states[i].state)
|
||||
{
|
||||
per_state[st_name] = 1 + (per_state[st_name]|0);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const st_name in per_state)
|
||||
{
|
||||
res += `vitastor_pg_count{pg_state="${st_name}",${pool_label}} ${per_state[st_name]}\n`;
|
||||
}
|
||||
const pool_active = per_state['active'] >= real_pg_count ? 1 : 0;
|
||||
res += `vitastor_pool_status{${pool_label}} ${pool_active}\n`;
|
||||
}
|
||||
|
||||
return res;
|
||||
}
|
||||
|
||||
function addslashes(str)
|
||||
{
|
||||
return ((str||'')+'').replace(/(["\n\\])/g, "\\$1"); // escape " \n \
|
||||
}
|
||||
|
||||
module.exports = { export_prometheus_metrics };
|
File diff suppressed because it is too large
Load Diff
|
@ -3,10 +3,10 @@
|
|||
|
||||
function derive_osd_stats(st, prev, prev_diff)
|
||||
{
|
||||
const diff = prev_diff || { op_stats: {}, subop_stats: {}, recovery_stats: {}, inode_stats: {} };
|
||||
const diff = { op_stats: {}, subop_stats: {}, recovery_stats: {}, inode_stats: {} };
|
||||
if (!st || !st.time || !prev || !prev.time || prev.time >= st.time)
|
||||
{
|
||||
return diff;
|
||||
return prev_diff || diff;
|
||||
}
|
||||
const timediff = BigInt(st.time*1000 - prev.time*1000);
|
||||
for (const op in st.op_stats||{})
|
||||
|
@ -74,7 +74,7 @@ function sum_op_stats(all_osd, prev_stats)
|
|||
);
|
||||
prev_stats.osd_stats[osd] = cur;
|
||||
}
|
||||
const sum_diff = { op_stats: {}, subop_stats: {}, recovery_stats: { degraded: {}, misplaced: {} } };
|
||||
const sum_diff = { op_stats: {}, subop_stats: {}, recovery_stats: {} };
|
||||
// Sum derived values instead of deriving summed
|
||||
for (const osd in all_osd.state)
|
||||
{
|
||||
|
|
Loading…
Reference in New Issue