2020-09-17 23:02:40 +03:00
|
|
|
// Copyright (c) Vitaliy Filippov, 2019+
|
2021-02-06 01:26:07 +03:00
|
|
|
// License: VNPL-1.1 (see README.md for details)
|
2020-09-17 23:02:40 +03:00
|
|
|
|
2021-11-28 01:19:42 +03:00
|
|
|
const fs = require('fs');
|
2020-05-12 17:49:18 +03:00
|
|
|
const http = require('http');
|
2020-09-01 18:50:23 +03:00
|
|
|
const crypto = require('crypto');
|
2020-05-12 17:49:18 +03:00
|
|
|
const os = require('os');
|
|
|
|
const WebSocket = require('ws');
|
2024-04-07 00:38:22 +03:00
|
|
|
const { RuleCombinator, parse_level_indexes, parse_pg_dsl } = require('./dsl_pgs.js');
|
|
|
|
const { SimpleCombinator, flatten_tree } = require('./simple_pgs.js');
|
2020-05-12 17:49:18 +03:00
|
|
|
const LPOptimizer = require('./lp-optimizer.js');
|
|
|
|
const stableStringify = require('./stable-stringify.js');
|
2020-09-01 02:22:50 +03:00
|
|
|
const PGUtil = require('./PGUtil.js');
|
2020-05-12 17:49:18 +03:00
|
|
|
|
2020-11-08 01:54:12 +03:00
|
|
|
// FIXME document all etcd keys and config variables in the form of JSON schema or similar
|
2020-03-12 00:56:41 +03:00
|
|
|
const etcd_nonempty_keys = {
|
|
|
|
'config/global': 1,
|
|
|
|
'config/node_placement': 1,
|
|
|
|
'config/pools': 1,
|
|
|
|
'config/pgs': 1,
|
|
|
|
'history/last_clean_pgs': 1,
|
|
|
|
'stats': 1,
|
|
|
|
};
|
2020-11-08 01:54:12 +03:00
|
|
|
const etcd_allow = new RegExp('^'+[
|
|
|
|
'config/global',
|
|
|
|
'config/node_placement',
|
|
|
|
'config/pools',
|
|
|
|
'config/osd/[1-9]\\d*',
|
|
|
|
'config/pgs',
|
2021-01-30 01:35:58 +03:00
|
|
|
'config/inode/[1-9]\\d*/[1-9]\\d*',
|
2020-11-08 01:54:12 +03:00
|
|
|
'osd/state/[1-9]\\d*',
|
|
|
|
'osd/stats/[1-9]\\d*',
|
2021-01-21 00:30:18 +03:00
|
|
|
'osd/inodestats/[1-9]\\d*',
|
2021-01-19 02:14:13 +03:00
|
|
|
'osd/space/[1-9]\\d*',
|
2020-11-08 01:54:12 +03:00
|
|
|
'mon/master',
|
2022-04-04 00:48:44 +03:00
|
|
|
'mon/member/[a-f0-9]+',
|
2020-11-08 01:54:12 +03:00
|
|
|
'pg/state/[1-9]\\d*/[1-9]\\d*',
|
|
|
|
'pg/stats/[1-9]\\d*/[1-9]\\d*',
|
|
|
|
'pg/history/[1-9]\\d*/[1-9]\\d*',
|
2021-11-06 13:38:52 +03:00
|
|
|
'pool/stats/[1-9]\\d*',
|
2021-03-06 01:30:42 +03:00
|
|
|
'history/last_clean_pgs',
|
2024-03-11 01:13:27 +03:00
|
|
|
'inode/stats/[1-9]\\d*/\\d+',
|
2021-10-21 14:00:54 +03:00
|
|
|
'pool/stats/[1-9]\\d*',
|
2020-11-08 01:54:12 +03:00
|
|
|
'stats',
|
2021-07-02 22:47:01 +03:00
|
|
|
'index/image/.*',
|
|
|
|
'index/maxid/[1-9]\\d*',
|
2020-11-08 01:54:12 +03:00
|
|
|
].join('$|^')+'$');
|
2020-09-01 17:07:06 +03:00
|
|
|
|
2020-11-08 01:54:12 +03:00
|
|
|
const etcd_tree = {
|
|
|
|
config: {
|
|
|
|
/* global: {
|
2021-04-30 01:21:16 +03:00
|
|
|
// WARNING: NOT ALL OF THESE ARE ACTUALLY CONFIGURABLE HERE
|
2021-06-20 00:23:56 +03:00
|
|
|
// THIS IS JUST A POOR MAN'S CONFIG DOCUMENTATION
|
2021-04-30 01:21:16 +03:00
|
|
|
// etcd connection
|
|
|
|
config_path: "/etc/vitastor/vitastor.conf",
|
|
|
|
etcd_prefix: "/vitastor",
|
2023-03-26 17:50:35 +03:00
|
|
|
// etcd connection - configurable online
|
|
|
|
etcd_address: "10.0.115.10:2379/v3",
|
2020-11-08 01:54:12 +03:00
|
|
|
// mon
|
2024-01-29 23:45:07 +03:00
|
|
|
etcd_mon_ttl: 5, // min: 1
|
2020-11-08 01:54:12 +03:00
|
|
|
etcd_mon_timeout: 1000, // ms. min: 0
|
|
|
|
etcd_mon_retries: 5, // min: 0
|
|
|
|
mon_change_timeout: 1000, // ms. min: 100
|
2023-12-28 01:50:26 +03:00
|
|
|
mon_retry_change_timeout: 50, // ms. min: 10
|
2020-11-08 01:54:12 +03:00
|
|
|
mon_stats_timeout: 1000, // ms. min: 100
|
2021-03-09 01:24:51 +03:00
|
|
|
osd_out_time: 600, // seconds. min: 0
|
2020-11-08 01:54:12 +03:00
|
|
|
placement_levels: { datacenter: 1, rack: 2, host: 3, osd: 4, ... },
|
2024-04-07 00:38:22 +03:00
|
|
|
use_old_pg_combinator: false,
|
2020-11-08 01:54:12 +03:00
|
|
|
// client and osd
|
2021-04-30 01:21:16 +03:00
|
|
|
tcp_header_buffer_size: 65536,
|
2020-11-08 01:54:12 +03:00
|
|
|
use_sync_send_recv: false,
|
2021-04-30 01:21:16 +03:00
|
|
|
use_rdma: true,
|
|
|
|
rdma_device: null, // for example, "rocep5s0f0"
|
|
|
|
rdma_port_num: 1,
|
|
|
|
rdma_gid_index: 0,
|
|
|
|
rdma_mtu: 4096,
|
|
|
|
rdma_max_sge: 128,
|
2023-02-28 02:58:28 +03:00
|
|
|
rdma_max_send: 8,
|
|
|
|
rdma_max_recv: 16,
|
2023-02-26 02:01:04 +03:00
|
|
|
rdma_max_msg: 132096,
|
2020-11-08 01:54:12 +03:00
|
|
|
block_size: 131072,
|
|
|
|
disk_alignment: 4096,
|
|
|
|
bitmap_granularity: 4096,
|
|
|
|
immediate_commit: false, // 'all' or 'small'
|
2023-08-12 20:26:40 +03:00
|
|
|
// client - configurable online
|
|
|
|
client_max_dirty_bytes: 33554432,
|
|
|
|
client_max_dirty_ops: 1024,
|
|
|
|
client_enable_writeback: false,
|
|
|
|
client_max_buffered_bytes: 33554432,
|
|
|
|
client_max_buffered_ops: 1024,
|
|
|
|
client_max_writeback_iodepth: 256,
|
2024-02-28 00:51:13 +03:00
|
|
|
client_retry_interval: 50, // ms. min: 10
|
|
|
|
client_eio_retry_interval: 1000, // ms
|
2024-04-07 18:58:44 +03:00
|
|
|
client_retry_enospc: true,
|
2024-04-07 19:39:51 +03:00
|
|
|
osd_nearfull_ratio: 0.95,
|
2023-03-26 17:50:35 +03:00
|
|
|
// client and osd - configurable online
|
|
|
|
log_level: 0,
|
2020-11-08 01:54:12 +03:00
|
|
|
peer_connect_interval: 5, // seconds. min: 1
|
|
|
|
peer_connect_timeout: 5, // seconds. min: 1
|
2021-03-11 00:36:23 +03:00
|
|
|
osd_idle_timeout: 5, // seconds. min: 1
|
|
|
|
osd_ping_timeout: 5, // seconds. min: 1
|
2022-01-23 00:00:00 +03:00
|
|
|
max_etcd_attempts: 5,
|
|
|
|
etcd_quick_timeout: 1000, // ms
|
|
|
|
etcd_slow_timeout: 5000, // ms
|
2022-01-26 02:38:00 +03:00
|
|
|
etcd_keepalive_timeout: 30, // seconds, default is max(30, etcd_report_interval*2)
|
2022-01-23 00:00:00 +03:00
|
|
|
etcd_ws_keepalive_interval: 30, // seconds
|
2022-01-26 02:38:00 +03:00
|
|
|
// osd
|
|
|
|
etcd_report_interval: 5, // seconds
|
2023-10-27 01:24:33 +03:00
|
|
|
etcd_stats_interval: 30, // seconds
|
2020-11-08 01:54:12 +03:00
|
|
|
run_primary: true,
|
2021-11-23 21:59:22 +03:00
|
|
|
osd_network: null, // "192.168.7.0/24" or an array of masks
|
2020-11-08 01:54:12 +03:00
|
|
|
bind_address: "0.0.0.0",
|
|
|
|
bind_port: 0,
|
2023-03-26 17:50:35 +03:00
|
|
|
readonly: false,
|
|
|
|
osd_memlock: false,
|
|
|
|
// osd - configurable online
|
2020-11-08 01:54:12 +03:00
|
|
|
autosync_interval: 5,
|
2021-10-30 14:26:48 +03:00
|
|
|
autosync_writes: 128,
|
2020-11-08 01:54:12 +03:00
|
|
|
client_queue_depth: 128, // unused
|
2023-12-09 15:39:17 +03:00
|
|
|
recovery_queue_depth: 1,
|
|
|
|
recovery_sleep_us: 0,
|
2023-12-30 19:43:27 +03:00
|
|
|
recovery_tune_util_low: 0.1,
|
|
|
|
recovery_tune_client_util_low: 0,
|
|
|
|
recovery_tune_util_high: 1.0,
|
|
|
|
recovery_tune_client_util_high: 0.5,
|
2023-12-09 15:39:17 +03:00
|
|
|
recovery_tune_interval: 1,
|
2023-12-20 02:24:04 +03:00
|
|
|
recovery_tune_agg_interval: 10, // 10 times recovery_tune_interval
|
2023-12-09 15:39:17 +03:00
|
|
|
recovery_tune_sleep_min_us: 10, // 10 microseconds
|
2023-04-21 02:18:37 +03:00
|
|
|
recovery_pg_switch: 128,
|
2021-03-15 02:26:39 +03:00
|
|
|
recovery_sync_batch: 16,
|
2021-03-11 00:36:23 +03:00
|
|
|
no_recovery: false,
|
|
|
|
no_rebalance: false,
|
2020-11-08 01:54:12 +03:00
|
|
|
print_stats_interval: 3,
|
2021-03-11 00:36:23 +03:00
|
|
|
slow_log_interval: 10,
|
2022-06-04 00:37:52 +03:00
|
|
|
inode_vanish_time: 60,
|
2023-04-18 02:08:43 +03:00
|
|
|
auto_scrub: false,
|
2023-05-21 12:52:30 +03:00
|
|
|
no_scrub: false,
|
2023-02-21 00:21:23 +03:00
|
|
|
scrub_interval: '30d', // 1s/1m/1h/1d
|
|
|
|
scrub_queue_depth: 1,
|
|
|
|
scrub_sleep: 0, // milliseconds
|
|
|
|
scrub_list_limit: 1000, // objects to list on one scrub iteration
|
2023-05-21 12:15:37 +03:00
|
|
|
scrub_find_best: true,
|
2023-04-10 01:05:41 +03:00
|
|
|
scrub_ec_max_bruteforce: 100, // maximum EC error locator brute-force iterators
|
2020-11-08 01:54:12 +03:00
|
|
|
// blockstore - fixed in superblock
|
|
|
|
block_size,
|
|
|
|
disk_alignment,
|
|
|
|
journal_block_size,
|
|
|
|
meta_block_size,
|
|
|
|
bitmap_granularity,
|
|
|
|
journal_device,
|
|
|
|
journal_offset,
|
|
|
|
journal_size,
|
|
|
|
disable_journal_fsync,
|
|
|
|
data_device,
|
|
|
|
data_offset,
|
|
|
|
data_size,
|
|
|
|
disable_data_fsync,
|
|
|
|
meta_device,
|
|
|
|
meta_offset,
|
|
|
|
disable_meta_fsync,
|
|
|
|
disable_device_lock,
|
2023-03-26 17:50:35 +03:00
|
|
|
// blockstore - configurable offline
|
2020-11-08 01:54:12 +03:00
|
|
|
inmemory_metadata,
|
|
|
|
inmemory_journal,
|
|
|
|
journal_sector_buffer_count,
|
|
|
|
journal_no_same_sector_overwrites,
|
2023-03-26 17:50:35 +03:00
|
|
|
// blockstore - configurable online
|
|
|
|
max_write_iodepth,
|
|
|
|
min_flusher_count: 1,
|
|
|
|
max_flusher_count: 256,
|
2022-01-26 02:38:00 +03:00
|
|
|
throttle_small_writes: false,
|
|
|
|
throttle_target_iops: 100,
|
|
|
|
throttle_target_mbs: 100,
|
|
|
|
throttle_target_parallelism: 1,
|
|
|
|
throttle_threshold_us: 50,
|
2020-11-08 01:54:12 +03:00
|
|
|
}, */
|
|
|
|
global: {},
|
|
|
|
/* node_placement: {
|
|
|
|
host1: { level: 'host', parent: 'rack1' },
|
|
|
|
...
|
|
|
|
}, */
|
|
|
|
node_placement: {},
|
|
|
|
/* pools: {
|
|
|
|
<id>: {
|
|
|
|
name: 'testpool',
|
2022-06-03 15:36:58 +03:00
|
|
|
// 'ec' uses Reed-Solomon-Vandermonde codes, 'jerasure' is an alias for 'ec'
|
|
|
|
scheme: 'replicated' | 'xor' | 'ec' | 'jerasure',
|
2020-11-08 01:54:12 +03:00
|
|
|
pg_size: 3,
|
|
|
|
pg_minsize: 2,
|
2022-06-03 15:36:58 +03:00
|
|
|
// number of parity chunks, required for EC
|
2020-11-30 00:08:25 +03:00
|
|
|
parity_chunks?: 1,
|
2020-11-08 01:54:12 +03:00
|
|
|
pg_count: 100,
|
2024-04-07 00:38:22 +03:00
|
|
|
// default is failure_domain=host
|
|
|
|
failure_domain?: 'host',
|
|
|
|
// additional failure domain rules; failure_domain=x is equivalent to x=123..N
|
|
|
|
level_placement?: 'dc=112233 host=123456',
|
|
|
|
raw_placement?: 'any, dc=1 host!=1, dc=1 host!=(1,2)',
|
|
|
|
old_combinator: false,
|
2020-11-08 01:54:12 +03:00
|
|
|
max_osd_combinations: 10000,
|
2022-08-09 02:27:02 +03:00
|
|
|
// block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
|
2022-09-03 11:16:33 +03:00
|
|
|
block_size: 131072,
|
2022-08-09 02:27:02 +03:00
|
|
|
bitmap_granularity: 4096,
|
|
|
|
// 'all'/'small'/'none', same as in OSD options
|
|
|
|
immediate_commit: 'none',
|
|
|
|
pg_stripe_size: 0,
|
2020-11-08 01:54:12 +03:00
|
|
|
root_node?: 'rack1',
|
|
|
|
// restrict pool to OSDs having all of these tags
|
|
|
|
osd_tags?: 'nvme' | [ 'nvme', ... ],
|
2022-05-07 00:59:57 +03:00
|
|
|
// prefer to put primary on OSD with these tags
|
|
|
|
primary_affinity_tags?: 'nvme' | [ 'nvme', ... ],
|
2023-02-21 00:21:23 +03:00
|
|
|
// scrub interval
|
|
|
|
scrub_interval?: '30d',
|
2020-09-01 02:13:03 +03:00
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
...
|
|
|
|
}, */
|
|
|
|
pools: {},
|
|
|
|
osd: {
|
2024-04-16 02:19:55 +03:00
|
|
|
/* <id>: { reweight?: 1, tags?: [ 'nvme', ... ], noout?: true }, ... */
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
/* pgs: {
|
|
|
|
hash: string,
|
|
|
|
items: {
|
|
|
|
<pool_id>: {
|
|
|
|
<pg_id>: {
|
|
|
|
osd_set: [ 1, 2, 3 ],
|
|
|
|
primary: 1,
|
|
|
|
pause: false,
|
2020-09-01 16:29:42 +03:00
|
|
|
}
|
|
|
|
}
|
2020-11-08 01:54:12 +03:00
|
|
|
}
|
|
|
|
}, */
|
|
|
|
pgs: {},
|
2021-01-30 01:35:58 +03:00
|
|
|
/* inode: {
|
|
|
|
<pool_id>: {
|
|
|
|
<inode_t>: {
|
|
|
|
name: string,
|
2021-02-10 01:21:57 +03:00
|
|
|
size?: uint64_t, // bytes
|
2021-01-30 01:35:58 +03:00
|
|
|
parent_pool?: <pool_id>,
|
|
|
|
parent_id?: <inode_t>,
|
|
|
|
readonly?: boolean,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}, */
|
|
|
|
inode: {},
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
osd: {
|
|
|
|
state: {
|
|
|
|
/* <osd_num_t>: {
|
|
|
|
state: "up",
|
|
|
|
addresses: string[],
|
|
|
|
host: string,
|
|
|
|
port: uint16_t,
|
|
|
|
primary_enabled: boolean,
|
|
|
|
blockstore_enabled: boolean,
|
2020-09-01 16:29:42 +03:00
|
|
|
}, */
|
2020-05-12 17:49:18 +03:00
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
stats: {
|
|
|
|
/* <osd_num_t>: {
|
|
|
|
time: number, // unix time
|
|
|
|
blockstore_ready: boolean,
|
|
|
|
size: uint64_t, // bytes
|
|
|
|
free: uint64_t, // bytes
|
|
|
|
host: string,
|
|
|
|
op_stats: {
|
|
|
|
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
|
|
},
|
|
|
|
subop_stats: {
|
|
|
|
<string>: { count: uint64_t, usec: uint64_t },
|
|
|
|
},
|
|
|
|
recovery_stats: {
|
|
|
|
degraded: { count: uint64_t, bytes: uint64_t },
|
|
|
|
misplaced: { count: uint64_t, bytes: uint64_t },
|
|
|
|
},
|
|
|
|
}, */
|
2020-05-12 17:49:18 +03:00
|
|
|
},
|
2021-01-21 00:30:18 +03:00
|
|
|
inodestats: {
|
2022-04-21 02:52:10 +03:00
|
|
|
/* <pool_id>: {
|
|
|
|
<inode_t>: {
|
|
|
|
read: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
|
|
write: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
|
|
delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
|
|
|
|
},
|
2021-01-21 00:30:18 +03:00
|
|
|
}, */
|
|
|
|
},
|
2021-01-19 02:14:13 +03:00
|
|
|
space: {
|
|
|
|
/* <osd_num_t>: {
|
2022-04-21 02:52:10 +03:00
|
|
|
<pool_id>: {
|
|
|
|
<inode_t>: uint64_t, // bytes
|
|
|
|
},
|
2021-01-19 02:14:13 +03:00
|
|
|
}, */
|
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
mon: {
|
|
|
|
master: {
|
2022-04-04 00:48:44 +03:00
|
|
|
/* ip: [ string ], id: uint64_t */
|
|
|
|
},
|
|
|
|
standby: {
|
|
|
|
/* <uint64_t>: { ip: [ string ] }, */
|
2020-05-12 17:49:18 +03:00
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
pg: {
|
|
|
|
state: {
|
|
|
|
/* <pool_id>: {
|
|
|
|
<pg_id>: {
|
|
|
|
primary: osd_num_t,
|
2023-02-20 23:01:20 +03:00
|
|
|
state: ("starting"|"peering"|"incomplete"|"active"|"repeering"|"stopping"|"offline"|
|
2023-04-10 01:05:41 +03:00
|
|
|
"degraded"|"has_incomplete"|"has_degraded"|"has_misplaced"|"has_unclean"|
|
|
|
|
"has_invalid"|"has_inconsistent"|"has_corrupted"|"left_on_dead"|"scrubbing")[],
|
2020-11-08 01:54:12 +03:00
|
|
|
}
|
|
|
|
}, */
|
2020-05-12 17:49:18 +03:00
|
|
|
},
|
2020-09-01 18:50:23 +03:00
|
|
|
stats: {
|
2020-11-08 01:54:12 +03:00
|
|
|
/* <pool_id>: {
|
|
|
|
<pg_id>: {
|
|
|
|
object_count: uint64_t,
|
|
|
|
clean_count: uint64_t,
|
|
|
|
misplaced_count: uint64_t,
|
|
|
|
degraded_count: uint64_t,
|
|
|
|
incomplete_count: uint64_t,
|
|
|
|
write_osd_set: osd_num_t[],
|
|
|
|
},
|
2020-09-01 18:50:23 +03:00
|
|
|
}, */
|
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
history: {
|
|
|
|
/* <pool_id>: {
|
|
|
|
<pg_id>: {
|
|
|
|
osd_sets: osd_num_t[][],
|
|
|
|
all_peers: osd_num_t[],
|
2022-04-10 12:21:37 +03:00
|
|
|
epoch: uint64_t,
|
2023-04-18 02:08:43 +03:00
|
|
|
next_scrub: uint64_t,
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
}, */
|
|
|
|
},
|
|
|
|
},
|
2021-01-19 02:14:13 +03:00
|
|
|
inode: {
|
2021-01-21 00:30:18 +03:00
|
|
|
stats: {
|
2021-06-20 00:23:56 +03:00
|
|
|
/* <pool_id>: {
|
|
|
|
<inode_t>: {
|
|
|
|
raw_used: uint64_t, // raw used bytes on OSDs
|
2021-11-10 23:51:15 +03:00
|
|
|
read: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
|
|
write: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
|
|
|
delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
2021-06-20 00:23:56 +03:00
|
|
|
},
|
2021-01-19 02:14:13 +03:00
|
|
|
}, */
|
|
|
|
},
|
|
|
|
},
|
2021-07-02 22:47:01 +03:00
|
|
|
pool: {
|
|
|
|
stats: {
|
|
|
|
/* <pool_id>: {
|
|
|
|
used_raw_tb: float, // used raw space in the pool
|
|
|
|
total_raw_tb: float, // maximum amount of space in the pool
|
|
|
|
raw_to_usable: float, // raw to usable ratio
|
|
|
|
space_efficiency: float, // 0..1
|
|
|
|
} */
|
|
|
|
},
|
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
stats: {
|
|
|
|
/* op_stats: {
|
2021-11-10 23:51:15 +03:00
|
|
|
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
subop_stats: {
|
2021-11-10 23:51:15 +03:00
|
|
|
<string>: { count: uint64_t, usec: uint64_t, iops: uint64_t, lat: uint64_t },
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
recovery_stats: {
|
2021-11-10 23:51:15 +03:00
|
|
|
degraded: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
|
|
|
|
misplaced: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
|
2020-11-08 01:54:12 +03:00
|
|
|
},
|
|
|
|
object_counts: {
|
|
|
|
object: uint64_t,
|
|
|
|
clean: uint64_t,
|
|
|
|
misplaced: uint64_t,
|
|
|
|
degraded: uint64_t,
|
|
|
|
incomplete: uint64_t,
|
2022-08-09 02:27:02 +03:00
|
|
|
},
|
|
|
|
object_bytes: {
|
|
|
|
total: uint64_t,
|
|
|
|
clean: uint64_t,
|
|
|
|
misplaced: uint64_t,
|
|
|
|
degraded: uint64_t,
|
|
|
|
incomplete: uint64_t,
|
2020-11-08 01:54:12 +03:00
|
|
|
}, */
|
|
|
|
},
|
2021-03-06 01:30:42 +03:00
|
|
|
history: {
|
|
|
|
last_clean_pgs: {},
|
|
|
|
},
|
2021-07-02 22:47:01 +03:00
|
|
|
index: {
|
|
|
|
image: {
|
|
|
|
/* <name>: {
|
|
|
|
id: uint64_t,
|
|
|
|
pool_id: uint64_t,
|
|
|
|
}, */
|
|
|
|
},
|
|
|
|
maxid: {
|
|
|
|
/* <pool_id>: uint64_t, */
|
|
|
|
},
|
|
|
|
},
|
2020-11-08 01:54:12 +03:00
|
|
|
};
|
2020-05-12 17:49:18 +03:00
|
|
|
|
2020-11-08 01:54:12 +03:00
|
|
|
// FIXME Split into several files
|
|
|
|
class Mon
|
|
|
|
{
|
2020-05-12 17:49:18 +03:00
|
|
|
constructor(config)
|
|
|
|
{
|
2024-01-17 00:16:56 +03:00
|
|
|
this.failconnect = (e) => this._die(e, 2);
|
|
|
|
this.die = (e) => this._die(e, 1);
|
2021-11-28 01:19:42 +03:00
|
|
|
if (fs.existsSync(config.config_path||'/etc/vitastor/vitastor.conf'))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
config = {
|
|
|
|
...JSON.parse(fs.readFileSync(config.config_path||'/etc/vitastor/vitastor.conf', { encoding: 'utf-8' })),
|
|
|
|
...config,
|
|
|
|
};
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2021-11-28 01:19:42 +03:00
|
|
|
this.parse_etcd_addresses(config.etcd_address||config.etcd_url);
|
2020-09-11 18:49:03 +03:00
|
|
|
this.verbose = config.verbose || 0;
|
2021-11-28 01:19:42 +03:00
|
|
|
this.initConfig = config;
|
2023-12-22 02:25:04 +03:00
|
|
|
this.config = { ...config };
|
2020-08-03 23:50:50 +03:00
|
|
|
this.etcd_prefix = config.etcd_prefix || '/vitastor';
|
2020-05-14 19:29:35 +03:00
|
|
|
this.etcd_prefix = this.etcd_prefix.replace(/\/\/+/g, '/').replace(/^\/?(.*[^\/])\/?$/, '/$1');
|
2020-05-12 17:49:18 +03:00
|
|
|
this.etcd_start_timeout = (config.etcd_start_timeout || 5) * 1000;
|
2020-09-01 17:07:06 +03:00
|
|
|
this.state = JSON.parse(JSON.stringify(this.constructor.etcd_tree));
|
2023-10-29 01:26:32 +03:00
|
|
|
this.prev_stats = { osd_stats: {}, osd_diff: {} };
|
2021-11-06 13:38:52 +03:00
|
|
|
this.signals_set = false;
|
2022-01-05 17:28:51 +03:00
|
|
|
this.ws = null;
|
|
|
|
this.ws_alive = false;
|
|
|
|
this.ws_keepalive_timer = null;
|
2021-11-28 01:43:31 +03:00
|
|
|
this.on_stop_cb = () => this.on_stop(0).catch(console.error);
|
2023-11-04 20:59:56 +03:00
|
|
|
this.recheck_pgs_active = false;
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
2021-11-28 01:19:42 +03:00
|
|
|
parse_etcd_addresses(addrs)
|
|
|
|
{
|
|
|
|
const is_local_ip = this.local_ips(true).reduce((a, c) => { a[c] = true; return a; }, {});
|
|
|
|
this.etcd_local = [];
|
|
|
|
this.etcd_urls = [];
|
|
|
|
this.selected_etcd_url = null;
|
|
|
|
this.etcd_urls_to_try = [];
|
|
|
|
if (!(addrs instanceof Array))
|
|
|
|
addrs = addrs ? (''+(addrs||'')).split(/,/) : [];
|
|
|
|
if (!addrs.length)
|
|
|
|
{
|
|
|
|
console.error('Vitastor etcd address(es) not specified. Please set on the command line or in the config file');
|
|
|
|
process.exit(1);
|
|
|
|
}
|
|
|
|
for (let url of addrs)
|
|
|
|
{
|
|
|
|
let scheme = 'http';
|
|
|
|
url = url.trim().replace(/^(https?):\/\//, (m, m1) => { scheme = m1; return ''; });
|
|
|
|
const slash = url.indexOf('/');
|
|
|
|
const colon = url.indexOf(':');
|
|
|
|
const is_local = is_local_ip[colon >= 0 ? url.substr(0, colon) : (slash >= 0 ? url.substr(0, slash) : url)];
|
|
|
|
url = scheme+'://'+(slash >= 0 ? url : url+'/v3');
|
|
|
|
if (is_local)
|
|
|
|
this.etcd_local.push(url);
|
|
|
|
else
|
|
|
|
this.etcd_urls.push(url);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-05-12 17:49:18 +03:00
|
|
|
async start()
|
|
|
|
{
|
|
|
|
await this.load_config();
|
|
|
|
await this.get_lease();
|
|
|
|
await this.become_master();
|
|
|
|
await this.load_cluster_state();
|
2020-09-05 02:14:43 +03:00
|
|
|
await this.start_watcher(this.config.etcd_mon_retries);
|
2021-11-06 13:38:52 +03:00
|
|
|
for (const pool_id in this.state.config.pools)
|
|
|
|
{
|
|
|
|
if (!this.state.pool.stats[pool_id] ||
|
2022-01-23 20:14:39 +03:00
|
|
|
!Number(this.state.pool.stats[pool_id].pg_real_size))
|
2021-11-06 13:38:52 +03:00
|
|
|
{
|
|
|
|
// Generate missing data in etcd
|
|
|
|
this.state.config.pgs.hash = null;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
await this.recheck_pgs();
|
2021-11-09 01:41:22 +03:00
|
|
|
this.schedule_update_stats();
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async load_config()
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
const res = await this.etcd_call('/kv/txn', { success: [
|
2020-05-12 17:49:18 +03:00
|
|
|
{ requestRange: { key: b64(this.etcd_prefix+'/config/global') } }
|
|
|
|
] }, this.etcd_start_timeout, -1);
|
2021-01-31 22:49:36 +03:00
|
|
|
if (res.responses[0].response_range.kvs)
|
|
|
|
{
|
|
|
|
this.parse_kv(res.responses[0].response_range.kvs[0]);
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
this.check_config();
|
|
|
|
}
|
|
|
|
|
|
|
|
check_config()
|
|
|
|
{
|
2024-01-29 23:45:07 +03:00
|
|
|
this.config.etcd_mon_ttl = Number(this.config.etcd_mon_ttl) || 5;
|
|
|
|
if (this.config.etcd_mon_ttl < 1)
|
2020-09-01 02:13:03 +03:00
|
|
|
{
|
2024-01-29 23:45:07 +03:00
|
|
|
this.config.etcd_mon_ttl = 1;
|
2020-09-01 02:13:03 +03:00
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
this.config.etcd_mon_timeout = Number(this.config.etcd_mon_timeout) || 0;
|
|
|
|
if (this.config.etcd_mon_timeout <= 0)
|
|
|
|
{
|
|
|
|
this.config.etcd_mon_timeout = 1000;
|
|
|
|
}
|
|
|
|
this.config.etcd_mon_retries = Number(this.config.etcd_mon_retries) || 5;
|
|
|
|
if (this.config.etcd_mon_retries < 0)
|
|
|
|
{
|
|
|
|
this.config.etcd_mon_retries = 0;
|
|
|
|
}
|
|
|
|
this.config.mon_change_timeout = Number(this.config.mon_change_timeout) || 1000;
|
|
|
|
if (this.config.mon_change_timeout < 100)
|
|
|
|
{
|
|
|
|
this.config.mon_change_timeout = 100;
|
|
|
|
}
|
2023-12-28 01:50:26 +03:00
|
|
|
this.config.mon_retry_change_timeout = Number(this.config.mon_retry_change_timeout) || 50;
|
|
|
|
if (this.config.mon_retry_change_timeout < 50)
|
|
|
|
{
|
|
|
|
this.config.mon_retry_change_timeout = 50;
|
|
|
|
}
|
2020-05-15 01:28:44 +03:00
|
|
|
this.config.mon_stats_timeout = Number(this.config.mon_stats_timeout) || 1000;
|
|
|
|
if (this.config.mon_stats_timeout < 100)
|
|
|
|
{
|
|
|
|
this.config.mon_stats_timeout = 100;
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
// After this number of seconds, a dead OSD will be removed from PG distribution
|
|
|
|
this.config.osd_out_time = Number(this.config.osd_out_time) || 0;
|
|
|
|
if (!this.config.osd_out_time)
|
|
|
|
{
|
2021-03-09 01:24:51 +03:00
|
|
|
this.config.osd_out_time = 600; // 10 minutes by default
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-28 01:19:42 +03:00
|
|
|
pick_next_etcd()
|
|
|
|
{
|
|
|
|
if (this.selected_etcd_url)
|
|
|
|
return this.selected_etcd_url;
|
|
|
|
if (!this.etcd_urls_to_try || !this.etcd_urls_to_try.length)
|
|
|
|
{
|
|
|
|
this.etcd_urls_to_try = [ ...this.etcd_local ];
|
|
|
|
const others = [ ...this.etcd_urls ];
|
|
|
|
while (others.length)
|
|
|
|
{
|
|
|
|
const url = others.splice(0|(others.length*Math.random()), 1);
|
|
|
|
this.etcd_urls_to_try.push(url[0]);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
this.selected_etcd_url = this.etcd_urls_to_try.shift();
|
|
|
|
return this.selected_etcd_url;
|
|
|
|
}
|
|
|
|
|
|
|
|
restart_watcher(cur_addr)
|
|
|
|
{
|
2022-01-05 17:28:51 +03:00
|
|
|
if (this.ws)
|
|
|
|
{
|
|
|
|
this.ws.close();
|
|
|
|
this.ws = null;
|
|
|
|
}
|
|
|
|
if (this.ws_keepalive_timer)
|
|
|
|
{
|
|
|
|
clearInterval(this.ws_keepalive_timer);
|
|
|
|
this.ws_keepalive_timer = null;
|
|
|
|
}
|
2021-11-28 01:19:42 +03:00
|
|
|
if (this.selected_etcd_url == cur_addr)
|
2022-01-05 17:28:51 +03:00
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
this.selected_etcd_url = null;
|
2022-01-05 17:28:51 +03:00
|
|
|
}
|
2021-11-28 01:19:42 +03:00
|
|
|
this.start_watcher(this.config.etcd_mon_retries).catch(this.die);
|
|
|
|
}
|
|
|
|
|
2020-05-12 17:49:18 +03:00
|
|
|
async start_watcher(retries)
|
|
|
|
{
|
|
|
|
let retry = 0;
|
2020-09-05 02:14:43 +03:00
|
|
|
if (!retries || retries < 1)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
retries = 1;
|
|
|
|
}
|
2023-08-03 23:56:50 +03:00
|
|
|
const tried = {};
|
2020-05-12 17:49:18 +03:00
|
|
|
while (retries < 0 || retry < retries)
|
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
const cur_addr = this.pick_next_etcd();
|
|
|
|
const base = 'ws'+cur_addr.substr(4);
|
2023-08-03 23:56:50 +03:00
|
|
|
let now = Date.now();
|
2023-10-29 01:30:40 +03:00
|
|
|
if (tried[base] && now-tried[base] < this.etcd_start_timeout)
|
2023-08-03 23:56:50 +03:00
|
|
|
{
|
2023-10-29 01:30:40 +03:00
|
|
|
await new Promise(ok => setTimeout(ok, this.etcd_start_timeout-(now-tried[base])));
|
2023-08-03 23:56:50 +03:00
|
|
|
now = Date.now();
|
|
|
|
}
|
|
|
|
tried[base] = now;
|
2024-04-20 02:02:13 +03:00
|
|
|
const ok = await new Promise(ok =>
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
const timer_id = setTimeout(() =>
|
|
|
|
{
|
|
|
|
this.ws.close();
|
2022-01-05 17:28:51 +03:00
|
|
|
this.ws = null;
|
2020-05-12 17:49:18 +03:00
|
|
|
ok(false);
|
2020-09-01 18:50:23 +03:00
|
|
|
}, this.config.etcd_mon_timeout);
|
2020-05-12 17:49:18 +03:00
|
|
|
this.ws = new WebSocket(base+'/watch');
|
2021-03-09 01:46:57 +03:00
|
|
|
const fail = () =>
|
|
|
|
{
|
|
|
|
ok(false);
|
|
|
|
};
|
|
|
|
this.ws.on('error', fail);
|
2020-05-12 17:49:18 +03:00
|
|
|
this.ws.on('open', () =>
|
|
|
|
{
|
2021-03-09 01:46:57 +03:00
|
|
|
this.ws.removeListener('error', fail);
|
2020-05-12 17:49:18 +03:00
|
|
|
if (timer_id)
|
|
|
|
clearTimeout(timer_id);
|
|
|
|
ok(true);
|
|
|
|
});
|
|
|
|
});
|
2020-09-05 02:14:43 +03:00
|
|
|
if (ok)
|
|
|
|
break;
|
2021-11-28 01:19:42 +03:00
|
|
|
if (this.selected_etcd_url == cur_addr)
|
|
|
|
this.selected_etcd_url = null;
|
2020-09-05 02:14:43 +03:00
|
|
|
this.ws = null;
|
2020-05-12 17:49:18 +03:00
|
|
|
retry++;
|
|
|
|
}
|
|
|
|
if (!this.ws)
|
|
|
|
{
|
2024-01-17 00:16:56 +03:00
|
|
|
this.failconnect('Failed to open etcd watch websocket');
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2021-11-28 01:19:42 +03:00
|
|
|
const cur_addr = this.selected_etcd_url;
|
2022-01-05 17:28:51 +03:00
|
|
|
this.ws_alive = true;
|
|
|
|
this.ws_keepalive_timer = setInterval(() =>
|
|
|
|
{
|
|
|
|
if (this.ws_alive)
|
|
|
|
{
|
|
|
|
this.ws_alive = false;
|
|
|
|
this.ws.send(JSON.stringify({ progress_request: {} }));
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
console.log('etcd websocket timed out, restarting it');
|
|
|
|
this.restart_watcher(cur_addr);
|
|
|
|
}
|
2023-12-23 20:07:29 +03:00
|
|
|
}, (Number(this.config.etcd_ws_keepalive_interval) || 30)*1000);
|
2021-11-28 01:19:42 +03:00
|
|
|
this.ws.on('error', () => this.restart_watcher(cur_addr));
|
2020-05-12 17:49:18 +03:00
|
|
|
this.ws.send(JSON.stringify({
|
|
|
|
create_request: {
|
|
|
|
key: b64(this.etcd_prefix+'/'),
|
|
|
|
range_end: b64(this.etcd_prefix+'0'),
|
2020-05-14 19:29:35 +03:00
|
|
|
start_revision: ''+this.etcd_watch_revision,
|
2020-05-12 17:49:18 +03:00
|
|
|
watch_id: 1,
|
2020-11-17 16:29:42 +03:00
|
|
|
progress_notify: true,
|
2020-05-12 17:49:18 +03:00
|
|
|
},
|
|
|
|
}));
|
|
|
|
this.ws.on('message', (msg) =>
|
|
|
|
{
|
2022-01-05 17:28:51 +03:00
|
|
|
this.ws_alive = true;
|
2020-05-12 17:49:18 +03:00
|
|
|
let data;
|
|
|
|
try
|
|
|
|
{
|
|
|
|
data = JSON.parse(msg);
|
|
|
|
}
|
|
|
|
catch (e)
|
|
|
|
{
|
|
|
|
}
|
2021-11-28 01:43:31 +03:00
|
|
|
if (!data || !data.result)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2021-11-28 01:43:31 +03:00
|
|
|
console.error('Unknown message received from watch websocket: '+msg);
|
|
|
|
}
|
|
|
|
else if (data.result.canceled)
|
|
|
|
{
|
|
|
|
// etcd watch canceled
|
|
|
|
if (data.result.compact_revision)
|
2020-09-05 02:14:43 +03:00
|
|
|
{
|
2021-11-28 01:43:31 +03:00
|
|
|
// we may miss events if we proceed
|
|
|
|
console.error('Revisions before '+data.result.compact_revision+' were compacted by etcd, exiting');
|
|
|
|
this.on_stop(1);
|
2020-09-05 02:14:43 +03:00
|
|
|
}
|
2021-11-28 01:43:31 +03:00
|
|
|
console.error('Watch canceled by etcd, reason: '+data.result.cancel_reason+', exiting');
|
|
|
|
this.on_stop(1);
|
|
|
|
}
|
|
|
|
else if (data.result.created)
|
|
|
|
{
|
|
|
|
// etcd watch created
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-03-06 01:30:42 +03:00
|
|
|
let stats_changed = false, changed = false, pg_states_changed = false;
|
2020-09-11 18:49:03 +03:00
|
|
|
if (this.verbose)
|
|
|
|
{
|
|
|
|
console.log('Revision '+data.result.header.revision+' events: ');
|
|
|
|
}
|
2020-09-05 02:14:43 +03:00
|
|
|
this.etcd_watch_revision = BigInt(data.result.header.revision)+BigInt(1);
|
2022-01-08 11:40:56 +03:00
|
|
|
for (const e of data.result.events||[])
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
this.parse_kv(e.kv);
|
2020-05-14 19:29:35 +03:00
|
|
|
const key = e.kv.key.substr(this.etcd_prefix.length);
|
2024-02-08 21:28:03 +03:00
|
|
|
if (key.substr(0, 11) == '/osd/state/')
|
|
|
|
{
|
|
|
|
stats_changed = true;
|
|
|
|
changed = true;
|
|
|
|
}
|
|
|
|
else if (key.substr(0, 11) == '/osd/stats/' || key.substr(0, 10) == '/pg/stats/' || key.substr(0, 16) == '/osd/inodestats/')
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
|
|
|
stats_changed = true;
|
|
|
|
}
|
2021-03-06 01:30:42 +03:00
|
|
|
else if (key.substr(0, 10) == '/pg/state/')
|
|
|
|
{
|
|
|
|
pg_states_changed = true;
|
|
|
|
}
|
2021-01-21 00:30:18 +03:00
|
|
|
else if (key != '/stats' && key.substr(0, 13) != '/inode/stats/')
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
|
|
|
changed = true;
|
|
|
|
}
|
2020-09-11 18:49:03 +03:00
|
|
|
if (this.verbose)
|
|
|
|
{
|
2021-02-28 19:51:14 +03:00
|
|
|
console.log(JSON.stringify(e));
|
2020-09-11 18:49:03 +03:00
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2021-03-06 01:30:42 +03:00
|
|
|
if (pg_states_changed)
|
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
this.save_last_clean().catch(this.die);
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
2020-05-15 01:28:44 +03:00
|
|
|
if (stats_changed)
|
|
|
|
{
|
|
|
|
this.schedule_update_stats();
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
if (changed)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-05-14 19:29:35 +03:00
|
|
|
this.schedule_recheck();
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2023-11-05 00:12:00 +03:00
|
|
|
// Schedule save_last_clean() to to run after a small timeout (1s) (to not spam etcd)
|
|
|
|
schedule_save_last_clean()
|
|
|
|
{
|
|
|
|
if (!this.save_last_clean_timer)
|
|
|
|
{
|
|
|
|
this.save_last_clean_timer = setTimeout(() =>
|
|
|
|
{
|
|
|
|
this.save_last_clean_timer = null;
|
|
|
|
this.save_last_clean().catch(this.die);
|
|
|
|
}, this.config.mon_change_timeout || 1000);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-03-06 01:30:42 +03:00
|
|
|
async save_last_clean()
|
|
|
|
{
|
2023-11-05 00:12:00 +03:00
|
|
|
if (this.save_last_clean_running)
|
|
|
|
{
|
|
|
|
this.schedule_save_last_clean();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
this.save_last_clean_running = true;
|
2021-03-06 01:30:42 +03:00
|
|
|
// last_clean_pgs is used to avoid extra data move when observing a series of changes in the cluster
|
2023-01-03 02:17:47 +03:00
|
|
|
const new_clean_pgs = { items: {} };
|
2024-04-16 02:38:32 +03:00
|
|
|
// eslint-disable-next-line indent
|
2023-01-03 02:17:47 +03:00
|
|
|
next_pool:
|
2021-03-06 01:30:42 +03:00
|
|
|
for (const pool_id in this.state.config.pools)
|
|
|
|
{
|
2023-01-03 02:17:47 +03:00
|
|
|
new_clean_pgs.items[pool_id] = (this.state.history.last_clean_pgs.items||{})[pool_id];
|
2021-03-06 01:30:42 +03:00
|
|
|
const pool_cfg = this.state.config.pools[pool_id];
|
|
|
|
if (!this.validate_pool_cfg(pool_id, pool_cfg, false))
|
|
|
|
{
|
2023-01-03 02:17:47 +03:00
|
|
|
continue next_pool;
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
|
|
|
for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
|
|
|
|
{
|
|
|
|
if (!this.state.pg.state[pool_id] ||
|
|
|
|
!this.state.pg.state[pool_id][pg_num] ||
|
|
|
|
!(this.state.pg.state[pool_id][pg_num].state instanceof Array))
|
|
|
|
{
|
|
|
|
// Unclean
|
2023-01-03 02:17:47 +03:00
|
|
|
continue next_pool;
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
|
|
|
let st = this.state.pg.state[pool_id][pg_num].state.join(',');
|
|
|
|
if (st != 'active' && st != 'active,left_on_dead' && st != 'left_on_dead,active')
|
|
|
|
{
|
|
|
|
// Unclean
|
2023-01-03 02:17:47 +03:00
|
|
|
continue next_pool;
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
|
|
|
}
|
2023-01-03 02:17:47 +03:00
|
|
|
new_clean_pgs.items[pool_id] = this.state.config.pgs.items[pool_id];
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
2023-01-03 02:17:47 +03:00
|
|
|
this.state.history.last_clean_pgs = new_clean_pgs;
|
2021-03-06 01:30:42 +03:00
|
|
|
await this.etcd_call('/kv/txn', {
|
|
|
|
success: [ { requestPut: {
|
|
|
|
key: b64(this.etcd_prefix+'/history/last_clean_pgs'),
|
|
|
|
value: b64(JSON.stringify(this.state.history.last_clean_pgs))
|
|
|
|
} } ],
|
|
|
|
}, this.etcd_start_timeout, 0);
|
2023-11-05 00:12:00 +03:00
|
|
|
this.save_last_clean_running = false;
|
2021-03-06 01:30:42 +03:00
|
|
|
}
|
|
|
|
|
2022-04-08 02:42:53 +03:00
|
|
|
get_mon_state()
|
|
|
|
{
|
|
|
|
return { ip: this.local_ips(), hostname: os.hostname() };
|
|
|
|
}
|
|
|
|
|
2020-05-12 17:49:18 +03:00
|
|
|
async get_lease()
|
|
|
|
{
|
|
|
|
const max_ttl = this.config.etcd_mon_ttl + this.config.etcd_mon_timeout/1000*this.config.etcd_mon_retries;
|
2022-04-04 00:48:44 +03:00
|
|
|
// Get lease
|
|
|
|
let res = await this.etcd_call('/lease/grant', { TTL: max_ttl }, this.config.etcd_mon_timeout, -1);
|
2020-05-12 17:49:18 +03:00
|
|
|
this.etcd_lease_id = res.ID;
|
2022-04-04 00:48:44 +03:00
|
|
|
// Register in /mon/member, just for the information
|
2022-04-08 02:42:53 +03:00
|
|
|
const state = this.get_mon_state();
|
2022-04-04 00:48:44 +03:00
|
|
|
res = await this.etcd_call('/kv/put', {
|
|
|
|
key: b64(this.etcd_prefix+'/mon/member/'+this.etcd_lease_id),
|
|
|
|
value: b64(JSON.stringify(state)),
|
|
|
|
lease: ''+this.etcd_lease_id
|
|
|
|
}, this.etcd_start_timeout, 0);
|
|
|
|
// Set refresh timer
|
2021-11-06 13:38:52 +03:00
|
|
|
this.lease_timer = setInterval(async () =>
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
const res = await this.etcd_call('/lease/keepalive', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
|
|
|
|
if (!res.result.TTL)
|
|
|
|
{
|
2024-01-17 00:16:56 +03:00
|
|
|
this.failconnect('Lease expired');
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2024-01-29 23:45:07 +03:00
|
|
|
}, this.config.etcd_mon_ttl*1000);
|
2021-11-06 13:38:52 +03:00
|
|
|
if (!this.signals_set)
|
|
|
|
{
|
|
|
|
process.on('SIGINT', this.on_stop_cb);
|
|
|
|
process.on('SIGTERM', this.on_stop_cb);
|
|
|
|
this.signals_set = true;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-11-28 01:43:31 +03:00
|
|
|
async on_stop(status)
|
2021-11-06 13:38:52 +03:00
|
|
|
{
|
|
|
|
clearInterval(this.lease_timer);
|
|
|
|
await this.etcd_call('/lease/revoke', { ID: this.etcd_lease_id }, this.config.etcd_mon_timeout, this.config.etcd_mon_retries);
|
2021-11-28 01:43:31 +03:00
|
|
|
process.exit(status);
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async become_master()
|
|
|
|
{
|
2022-04-08 02:42:53 +03:00
|
|
|
const state = { ...this.get_mon_state(), id: ''+this.etcd_lease_id };
|
2024-04-16 02:38:32 +03:00
|
|
|
// eslint-disable-next-line no-constant-condition
|
2020-05-12 17:49:18 +03:00
|
|
|
while (1)
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
const res = await this.etcd_call('/kv/txn', {
|
2020-05-12 17:49:18 +03:00
|
|
|
compare: [ { target: 'CREATE', create_revision: 0, key: b64(this.etcd_prefix+'/mon/master') } ],
|
2020-09-05 02:14:43 +03:00
|
|
|
success: [ { requestPut: { key: b64(this.etcd_prefix+'/mon/master'), value: b64(JSON.stringify(state)), lease: ''+this.etcd_lease_id } } ],
|
2020-05-12 17:49:18 +03:00
|
|
|
}, this.etcd_start_timeout, 0);
|
2020-09-05 02:14:43 +03:00
|
|
|
if (res.succeeded)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
break;
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2020-09-05 02:14:43 +03:00
|
|
|
console.log('Waiting to become master');
|
|
|
|
await new Promise(ok => setTimeout(ok, this.etcd_start_timeout));
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2020-09-05 02:14:43 +03:00
|
|
|
console.log('Became master');
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async load_cluster_state()
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
const res = await this.etcd_call('/kv/txn', { success: [
|
2020-05-12 17:49:18 +03:00
|
|
|
{ requestRange: { key: b64(this.etcd_prefix+'/'), range_end: b64(this.etcd_prefix+'0') } },
|
|
|
|
] }, this.etcd_start_timeout, -1);
|
|
|
|
this.etcd_watch_revision = BigInt(res.header.revision)+BigInt(1);
|
2020-09-05 02:14:43 +03:00
|
|
|
this.state = JSON.parse(JSON.stringify(this.constructor.etcd_tree));
|
2020-05-12 17:49:18 +03:00
|
|
|
for (const response of res.responses)
|
|
|
|
{
|
|
|
|
for (const kv of response.response_range.kvs)
|
|
|
|
{
|
|
|
|
this.parse_kv(kv);
|
|
|
|
}
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
all_osds()
|
|
|
|
{
|
|
|
|
return Object.keys(this.state.osd.stats);
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
get_osd_tree()
|
|
|
|
{
|
2020-09-01 02:13:03 +03:00
|
|
|
const levels = this.config.placement_levels||{};
|
2020-05-12 17:49:18 +03:00
|
|
|
levels.host = levels.host || 100;
|
|
|
|
levels.osd = levels.osd || 101;
|
2024-04-09 15:58:32 +03:00
|
|
|
const tree = {};
|
2020-09-13 12:23:11 +03:00
|
|
|
let up_osds = {};
|
2020-09-01 02:13:03 +03:00
|
|
|
for (const node_id in this.state.config.node_placement||{})
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-09-01 02:13:03 +03:00
|
|
|
const node_cfg = this.state.config.node_placement[node_id];
|
2021-12-12 01:25:45 +03:00
|
|
|
if (/^\d+$/.exec(node_id))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2021-12-12 01:25:45 +03:00
|
|
|
node_cfg.level = 'osd';
|
|
|
|
}
|
|
|
|
if (!node_id || !node_cfg.level || !levels[node_cfg.level])
|
|
|
|
{
|
|
|
|
// All nodes must have non-empty IDs and valid levels
|
2020-05-12 17:49:18 +03:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
tree[node_id] = { id: node_id, level: node_cfg.level, parent: node_cfg.parent, children: [] };
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
// This requires monitor system time to be in sync with OSD system times (at least to some extent)
|
2020-05-12 17:49:18 +03:00
|
|
|
const down_time = Date.now()/1000 - this.config.osd_out_time;
|
2020-05-14 19:29:35 +03:00
|
|
|
for (const osd_num of this.all_osds().sort((a, b) => a - b))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-05-14 19:29:35 +03:00
|
|
|
const stat = this.state.osd.stats[osd_num];
|
2024-04-16 02:19:55 +03:00
|
|
|
const osd_cfg = this.state.config.osd[osd_num];
|
|
|
|
if (stat && stat.size && (this.state.osd.state[osd_num] || Number(stat.time) >= down_time ||
|
|
|
|
osd_cfg && osd_cfg.noout))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
// Numeric IDs are reserved for OSDs
|
2024-04-05 23:14:11 +03:00
|
|
|
let reweight = osd_cfg == null ? 1 : Number(osd_cfg.reweight);
|
2020-09-11 23:59:30 +03:00
|
|
|
if (reweight < 0 || isNaN(reweight))
|
|
|
|
reweight = 1;
|
2020-09-13 12:23:11 +03:00
|
|
|
if (this.state.osd.state[osd_num] && reweight > 0)
|
|
|
|
{
|
|
|
|
// React to down OSDs immediately
|
|
|
|
up_osds[osd_num] = true;
|
|
|
|
}
|
2020-10-21 00:07:40 +03:00
|
|
|
tree[osd_num] = tree[osd_num] || {};
|
|
|
|
tree[osd_num].id = osd_num;
|
|
|
|
tree[osd_num].parent = tree[osd_num].parent || stat.host;
|
2020-05-12 17:49:18 +03:00
|
|
|
tree[osd_num].level = 'osd';
|
2020-05-22 12:52:27 +03:00
|
|
|
tree[osd_num].size = reweight * stat.size / 1024 / 1024 / 1024 / 1024; // terabytes
|
2020-10-04 17:20:09 +03:00
|
|
|
if (osd_cfg && osd_cfg.tags)
|
|
|
|
{
|
|
|
|
tree[osd_num].tags = (osd_cfg.tags instanceof Array ? [ ...osd_cfg.tags ] : [ osd_cfg.tags ])
|
|
|
|
.reduce((a, c) => { a[c] = true; return a; }, {});
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
delete tree[osd_num].children;
|
2021-12-12 01:25:45 +03:00
|
|
|
if (!tree[stat.host])
|
2020-10-21 00:07:40 +03:00
|
|
|
{
|
2021-12-12 01:25:45 +03:00
|
|
|
tree[stat.host] = {
|
|
|
|
id: stat.host,
|
2020-10-21 00:07:40 +03:00
|
|
|
level: 'host',
|
|
|
|
parent: null,
|
|
|
|
children: [],
|
|
|
|
};
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
}
|
2024-04-09 15:58:32 +03:00
|
|
|
return { up_osds, levels, osd_tree: tree };
|
|
|
|
}
|
|
|
|
|
|
|
|
make_hier_tree(tree)
|
|
|
|
{
|
|
|
|
const levels = this.config.placement_levels||{};
|
|
|
|
levels.host = levels.host || 100;
|
|
|
|
levels.osd = levels.osd || 101;
|
|
|
|
tree = { ...tree };
|
|
|
|
for (const node_id in tree)
|
|
|
|
{
|
|
|
|
tree[node_id] = { ...tree[node_id], children: [] };
|
|
|
|
}
|
|
|
|
tree[''] = { children: [] };
|
2020-05-12 17:49:18 +03:00
|
|
|
for (const node_id in tree)
|
|
|
|
{
|
2023-05-17 01:20:30 +03:00
|
|
|
if (node_id === '' || tree[node_id].level === 'osd' && (!tree[node_id].size || tree[node_id].size <= 0))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
const node_cfg = tree[node_id];
|
|
|
|
const node_level = levels[node_cfg.level] || node_cfg.level;
|
|
|
|
let parent_level = node_cfg.parent && tree[node_cfg.parent] && tree[node_cfg.parent].children
|
|
|
|
&& tree[node_cfg.parent].level;
|
|
|
|
parent_level = parent_level ? (levels[parent_level] || parent_level) : null;
|
|
|
|
// Parent's level must be less than child's; OSDs must be leaves
|
2020-10-21 00:07:40 +03:00
|
|
|
const parent = parent_level && parent_level < node_level ? node_cfg.parent : '';
|
2020-05-12 17:49:18 +03:00
|
|
|
tree[parent].children.push(tree[node_id]);
|
|
|
|
}
|
2024-04-20 02:02:28 +03:00
|
|
|
// Delete empty nodes
|
|
|
|
let deleted = 0;
|
|
|
|
do
|
|
|
|
{
|
|
|
|
deleted = 0;
|
|
|
|
for (const node_id in tree)
|
|
|
|
{
|
|
|
|
if (tree[node_id].level !== 'osd' && (!tree[node_id].children || !tree[node_id].children.length))
|
|
|
|
{
|
|
|
|
const parent = tree[node_id].parent;
|
|
|
|
if (parent)
|
|
|
|
{
|
|
|
|
tree[parent].children = tree[parent].children.filter(c => c != tree[node_id]);
|
|
|
|
}
|
|
|
|
deleted++;
|
|
|
|
delete tree[node_id];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
} while (deleted > 0);
|
2024-04-09 15:58:32 +03:00
|
|
|
return tree;
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
2020-09-01 18:50:23 +03:00
|
|
|
async stop_all_pgs(pool_id)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-05-14 19:29:35 +03:00
|
|
|
let has_online = false, paused = true;
|
2020-09-01 18:50:23 +03:00
|
|
|
for (const pg in this.state.config.pgs.items[pool_id]||{})
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
// FIXME: Change all (||{}) to ?. (optional chaining) at some point
|
|
|
|
const cur_state = (((this.state.pg.state[pool_id]||{})[pg]||{}).state||[]).join(',');
|
2020-05-14 19:29:35 +03:00
|
|
|
if (cur_state != '' && cur_state != 'offline')
|
|
|
|
{
|
|
|
|
has_online = true;
|
|
|
|
}
|
2020-09-01 18:50:23 +03:00
|
|
|
if (!this.state.config.pgs.items[pool_id][pg].pause)
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
|
|
|
paused = false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (!paused)
|
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
console.log('Stopping all PGs for pool '+pool_id+' before changing PG count');
|
2020-05-14 19:29:35 +03:00
|
|
|
const new_cfg = JSON.parse(JSON.stringify(this.state.config.pgs));
|
2020-09-01 18:50:23 +03:00
|
|
|
for (const pg in new_cfg.items[pool_id])
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
new_cfg.items[pool_id][pg].pause = true;
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
|
|
|
// Check that no OSDs change their state before we pause PGs
|
|
|
|
// Doing this we make sure that OSDs don't wake up in the middle of our "transaction"
|
|
|
|
// and can't see the old PG configuration
|
|
|
|
const checks = [];
|
|
|
|
for (const osd_num of this.all_osds())
|
|
|
|
{
|
|
|
|
const key = b64(this.etcd_prefix+'/osd/state/'+osd_num);
|
|
|
|
checks.push({ key, target: 'MOD', result: 'LESS', mod_revision: ''+this.etcd_watch_revision });
|
|
|
|
}
|
2024-04-20 02:02:13 +03:00
|
|
|
await this.etcd_call('/kv/txn', {
|
2020-05-14 19:29:35 +03:00
|
|
|
compare: [
|
|
|
|
{ key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
|
|
|
|
{ key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
|
|
|
|
...checks,
|
|
|
|
],
|
|
|
|
success: [
|
|
|
|
{ requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_cfg)) } },
|
|
|
|
],
|
|
|
|
}, this.config.etcd_mon_timeout, 0);
|
2020-12-01 14:13:24 +03:00
|
|
|
return false;
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
|
|
|
return !has_online;
|
|
|
|
}
|
|
|
|
|
2021-03-14 01:26:06 +03:00
|
|
|
reset_rng()
|
|
|
|
{
|
|
|
|
this.seed = 0x5f020e43;
|
|
|
|
}
|
|
|
|
|
|
|
|
rng()
|
|
|
|
{
|
|
|
|
this.seed ^= this.seed << 13;
|
|
|
|
this.seed ^= this.seed >> 17;
|
|
|
|
this.seed ^= this.seed << 5;
|
|
|
|
return this.seed + 2147483648;
|
|
|
|
}
|
|
|
|
|
2022-05-07 00:59:57 +03:00
|
|
|
pick_primary(pool_id, osd_set, up_osds, aff_osds)
|
2021-03-14 01:26:06 +03:00
|
|
|
{
|
|
|
|
let alive_set;
|
|
|
|
if (this.state.config.pools[pool_id].scheme === 'replicated')
|
2022-05-07 00:59:57 +03:00
|
|
|
{
|
|
|
|
// Prefer "affinity" OSDs
|
|
|
|
alive_set = osd_set.filter(osd_num => osd_num && aff_osds[osd_num]);
|
|
|
|
if (!alive_set.length)
|
|
|
|
alive_set = osd_set.filter(osd_num => osd_num && up_osds[osd_num]);
|
|
|
|
}
|
2021-03-14 01:26:06 +03:00
|
|
|
else
|
|
|
|
{
|
|
|
|
// Prefer data OSDs for EC because they can actually read something without an additional network hop
|
|
|
|
const pg_data_size = (this.state.config.pools[pool_id].pg_size||0) -
|
|
|
|
(this.state.config.pools[pool_id].parity_chunks||0);
|
2022-05-07 00:59:57 +03:00
|
|
|
alive_set = osd_set.slice(0, pg_data_size).filter(osd_num => osd_num && aff_osds[osd_num]);
|
2021-03-14 01:26:06 +03:00
|
|
|
if (!alive_set.length)
|
2022-05-07 00:59:57 +03:00
|
|
|
alive_set = osd_set.filter(osd_num => osd_num && aff_osds[osd_num]);
|
|
|
|
if (!alive_set.length)
|
|
|
|
{
|
|
|
|
alive_set = osd_set.slice(0, pg_data_size).filter(osd_num => osd_num && up_osds[osd_num]);
|
|
|
|
if (!alive_set.length)
|
|
|
|
alive_set = osd_set.filter(osd_num => osd_num && up_osds[osd_num]);
|
|
|
|
}
|
2021-03-14 01:26:06 +03:00
|
|
|
}
|
|
|
|
if (!alive_set.length)
|
|
|
|
return 0;
|
|
|
|
return alive_set[this.rng() % alive_set.length];
|
|
|
|
}
|
|
|
|
|
2023-05-07 23:17:51 +03:00
|
|
|
save_new_pgs_txn(save_to, request, pool_id, up_osds, osd_tree, prev_pgs, new_pgs, pg_history)
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
2023-05-17 00:45:59 +03:00
|
|
|
const aff_osds = this.get_affinity_osds(this.state.config.pools[pool_id] || {}, up_osds, osd_tree);
|
2020-05-14 19:29:35 +03:00
|
|
|
const pg_items = {};
|
2021-03-14 01:26:06 +03:00
|
|
|
this.reset_rng();
|
2020-05-14 19:29:35 +03:00
|
|
|
new_pgs.map((osd_set, i) =>
|
|
|
|
{
|
|
|
|
osd_set = osd_set.map(osd_num => osd_num === LPOptimizer.NO_OSD ? 0 : osd_num);
|
|
|
|
pg_items[i+1] = {
|
|
|
|
osd_set,
|
2022-05-07 00:59:57 +03:00
|
|
|
primary: this.pick_primary(pool_id, osd_set, up_osds, aff_osds),
|
2020-05-14 19:29:35 +03:00
|
|
|
};
|
2020-03-12 00:02:18 +03:00
|
|
|
if (prev_pgs[i] && prev_pgs[i].join(' ') != osd_set.join(' ') &&
|
|
|
|
prev_pgs[i].filter(osd_num => osd_num).length > 0)
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
|
|
|
pg_history[i] = pg_history[i] || {};
|
|
|
|
pg_history[i].osd_sets = pg_history[i].osd_sets || [];
|
|
|
|
pg_history[i].osd_sets.push(prev_pgs[i]);
|
|
|
|
}
|
2021-03-27 18:16:00 +03:00
|
|
|
if (pg_history[i] && pg_history[i].osd_sets)
|
|
|
|
{
|
|
|
|
pg_history[i].osd_sets = Object.values(pg_history[i].osd_sets
|
|
|
|
.reduce((a, c) => { a[c.join(' ')] = c; return a; }, {}));
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
});
|
|
|
|
for (let i = 0; i < new_pgs.length || i < prev_pgs.length; i++)
|
|
|
|
{
|
2020-09-13 12:23:11 +03:00
|
|
|
// FIXME: etcd has max_txn_ops limit, and it's 128 by default
|
|
|
|
// Sooo we probably want to change our storage scheme for PG histories...
|
2020-09-01 18:50:23 +03:00
|
|
|
request.compare.push({
|
|
|
|
key: b64(this.etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
2020-05-14 19:29:35 +03:00
|
|
|
target: 'MOD',
|
|
|
|
mod_revision: ''+this.etcd_watch_revision,
|
|
|
|
result: 'LESS',
|
2020-05-12 17:49:18 +03:00
|
|
|
});
|
2020-05-14 19:29:35 +03:00
|
|
|
if (pg_history[i])
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
request.success.push({
|
2020-05-14 19:29:35 +03:00
|
|
|
requestPut: {
|
2020-09-01 18:50:23 +03:00
|
|
|
key: b64(this.etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
2020-05-14 19:29:35 +03:00
|
|
|
value: b64(JSON.stringify(pg_history[i])),
|
|
|
|
},
|
|
|
|
});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
request.success.push({
|
2020-05-14 19:29:35 +03:00
|
|
|
requestDeleteRange: {
|
2020-09-01 18:50:23 +03:00
|
|
|
key: b64(this.etcd_prefix+'/pg/history/'+pool_id+'/'+(i+1)),
|
2020-05-14 19:29:35 +03:00
|
|
|
},
|
|
|
|
});
|
|
|
|
}
|
|
|
|
}
|
2023-05-07 23:17:51 +03:00
|
|
|
save_to.items = save_to.items || {};
|
2020-12-01 14:13:24 +03:00
|
|
|
if (!new_pgs.length)
|
|
|
|
{
|
2023-05-07 23:17:51 +03:00
|
|
|
delete save_to.items[pool_id];
|
2020-12-01 14:13:24 +03:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2023-05-07 23:17:51 +03:00
|
|
|
save_to.items[pool_id] = pg_items;
|
2020-12-01 14:13:24 +03:00
|
|
|
}
|
2020-09-01 18:50:23 +03:00
|
|
|
}
|
|
|
|
|
2020-09-05 02:14:43 +03:00
|
|
|
validate_pool_cfg(pool_id, pool_cfg, warn)
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
|
|
|
pool_cfg.pg_size = Math.floor(pool_cfg.pg_size);
|
|
|
|
pool_cfg.pg_minsize = Math.floor(pool_cfg.pg_minsize);
|
2020-11-30 00:08:25 +03:00
|
|
|
pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
|
2020-09-01 18:50:23 +03:00
|
|
|
pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
|
|
|
|
pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
|
|
|
|
if (!/^[1-9]\d*$/.exec(''+pool_id))
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
|
|
|
console.log('Pool ID '+pool_id+' is invalid');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
2022-06-03 15:36:58 +03:00
|
|
|
if (pool_cfg.scheme !== 'xor' && pool_cfg.scheme !== 'replicated' &&
|
|
|
|
pool_cfg.scheme !== 'ec' && pool_cfg.scheme !== 'jerasure')
|
2020-11-30 00:08:25 +03:00
|
|
|
{
|
|
|
|
if (warn)
|
2022-06-03 15:36:58 +03:00
|
|
|
console.log('Pool '+pool_id+' has invalid coding scheme (one of "xor", "replicated", "ec" and "jerasure" required)');
|
2020-11-30 00:08:25 +03:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (!pool_cfg.pg_size || pool_cfg.pg_size < 1 || pool_cfg.pg_size > 256 ||
|
2022-06-03 15:36:58 +03:00
|
|
|
pool_cfg.scheme !== 'replicated' && pool_cfg.pg_size < 3)
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid pg_size');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
2020-09-02 21:54:32 +03:00
|
|
|
if (!pool_cfg.pg_minsize || pool_cfg.pg_minsize < 1 || pool_cfg.pg_minsize > pool_cfg.pg_size ||
|
|
|
|
pool_cfg.scheme === 'xor' && pool_cfg.pg_minsize < (pool_cfg.pg_size - 1))
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid pg_minsize');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
2020-11-30 00:08:25 +03:00
|
|
|
if (pool_cfg.scheme === 'xor' && pool_cfg.parity_chunks != 0 && pool_cfg.parity_chunks != 1)
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
2020-11-30 00:08:25 +03:00
|
|
|
console.log('Pool '+pool_id+' has invalid parity_chunks (must be 1)');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
2022-06-03 15:36:58 +03:00
|
|
|
if ((pool_cfg.scheme === 'ec' || pool_cfg.scheme === 'jerasure') &&
|
|
|
|
(pool_cfg.parity_chunks < 1 || pool_cfg.parity_chunks > pool_cfg.pg_size-2))
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
2020-11-30 00:08:25 +03:00
|
|
|
console.log('Pool '+pool_id+' has invalid parity_chunks (must be between 1 and pg_size-2)');
|
2020-09-02 21:54:32 +03:00
|
|
|
return false;
|
|
|
|
}
|
2020-11-30 00:08:25 +03:00
|
|
|
if (!pool_cfg.pg_count || pool_cfg.pg_count < 1)
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid pg_count');
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (!pool_cfg.name)
|
2020-09-02 21:54:32 +03:00
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
2020-11-30 00:08:25 +03:00
|
|
|
console.log('Pool '+pool_id+' has empty name');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (pool_cfg.max_osd_combinations < 100)
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid max_osd_combinations (must be at least 100)');
|
2020-09-01 18:50:23 +03:00
|
|
|
return false;
|
|
|
|
}
|
2020-10-04 17:20:09 +03:00
|
|
|
if (pool_cfg.root_node && typeof(pool_cfg.root_node) != 'string')
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid root_node (must be a string)');
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
if (pool_cfg.osd_tags && typeof(pool_cfg.osd_tags) != 'string' &&
|
|
|
|
(!(pool_cfg.osd_tags instanceof Array) || pool_cfg.osd_tags.filter(t => typeof t != 'string').length > 0))
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid osd_tags (must be a string or array of strings)');
|
|
|
|
return false;
|
|
|
|
}
|
2022-05-07 00:59:57 +03:00
|
|
|
if (pool_cfg.primary_affinity_tags && typeof(pool_cfg.primary_affinity_tags) != 'string' &&
|
|
|
|
(!(pool_cfg.primary_affinity_tags instanceof Array) || pool_cfg.primary_affinity_tags.filter(t => typeof t != 'string').length > 0))
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' has invalid primary_affinity_tags (must be a string or array of strings)');
|
|
|
|
return false;
|
|
|
|
}
|
2024-04-07 00:38:22 +03:00
|
|
|
if (!this.get_pg_rules(pool_id, pool_cfg, true))
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
2020-09-01 18:50:23 +03:00
|
|
|
return true;
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
|
|
|
|
2024-01-01 18:40:41 +03:00
|
|
|
filter_osds_by_root_node(pool_tree, root_node)
|
|
|
|
{
|
|
|
|
if (!root_node)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
2024-04-09 15:58:32 +03:00
|
|
|
let hier_tree = this.make_hier_tree(pool_tree);
|
|
|
|
let included = [ ...(hier_tree[root_node] || {}).children||[] ];
|
2024-04-09 15:48:31 +03:00
|
|
|
for (let i = 0; i < included.length; i++)
|
2024-01-01 18:40:41 +03:00
|
|
|
{
|
2024-04-09 15:48:31 +03:00
|
|
|
if (included[i].children)
|
2024-01-01 18:40:41 +03:00
|
|
|
{
|
2024-04-09 15:48:31 +03:00
|
|
|
included.splice(i+1, 0, ...included[i].children);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
let cur = pool_tree[root_node] || {};
|
2024-04-09 15:58:32 +03:00
|
|
|
while (cur && cur.id)
|
2024-04-09 15:48:31 +03:00
|
|
|
{
|
|
|
|
included.unshift(cur);
|
2024-04-09 15:58:32 +03:00
|
|
|
cur = pool_tree[cur.parent||''];
|
2024-04-09 15:48:31 +03:00
|
|
|
}
|
|
|
|
included = included.reduce((a, c) => { a[c.id||''] = true; return a; }, {});
|
|
|
|
for (const item in pool_tree)
|
|
|
|
{
|
|
|
|
if (!included[item])
|
|
|
|
{
|
|
|
|
delete pool_tree[item];
|
2024-01-01 18:40:41 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
filter_osds_by_tags(orig_tree, tags)
|
2020-10-04 17:20:09 +03:00
|
|
|
{
|
|
|
|
if (!tags)
|
|
|
|
{
|
2020-10-17 02:28:48 +03:00
|
|
|
return;
|
2020-10-04 17:20:09 +03:00
|
|
|
}
|
|
|
|
for (const tag of (tags instanceof Array ? tags : [ tags ]))
|
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
for (const osd in orig_tree)
|
2020-10-04 17:20:09 +03:00
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
if (orig_tree[osd].level === 'osd' &&
|
|
|
|
(!orig_tree[osd].tags || !orig_tree[osd].tags[tag]))
|
2020-10-04 17:20:09 +03:00
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
delete orig_tree[osd];
|
2020-10-04 17:20:09 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2024-01-01 18:40:41 +03:00
|
|
|
filter_osds_by_block_layout(orig_tree, block_size, bitmap_granularity, immediate_commit)
|
2023-09-02 17:55:53 +03:00
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
for (const osd in orig_tree)
|
2023-09-02 17:55:53 +03:00
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
if (orig_tree[osd].level === 'osd')
|
2023-09-02 17:55:53 +03:00
|
|
|
{
|
|
|
|
const osd_stat = this.state.osd.stats[osd];
|
|
|
|
if (osd_stat && (osd_stat.bs_block_size && osd_stat.bs_block_size != block_size ||
|
|
|
|
osd_stat.bitmap_granularity && osd_stat.bitmap_granularity != bitmap_granularity ||
|
|
|
|
osd_stat.immediate_commit == 'small' && immediate_commit == 'all' ||
|
|
|
|
osd_stat.immediate_commit == 'none' && immediate_commit != 'none'))
|
|
|
|
{
|
2024-04-09 14:50:47 +03:00
|
|
|
delete orig_tree[osd];
|
2023-09-02 17:55:53 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-07 00:59:57 +03:00
|
|
|
get_affinity_osds(pool_cfg, up_osds, osd_tree)
|
|
|
|
{
|
|
|
|
let aff_osds = up_osds;
|
|
|
|
if (pool_cfg.primary_affinity_tags)
|
|
|
|
{
|
2024-01-01 18:40:41 +03:00
|
|
|
aff_osds = Object.keys(up_osds).reduce((a, c) => { a[c] = osd_tree[c]; return a; }, {});
|
|
|
|
this.filter_osds_by_tags(aff_osds, pool_cfg.primary_affinity_tags);
|
|
|
|
for (const osd in aff_osds)
|
|
|
|
{
|
|
|
|
aff_osds[osd] = true;
|
|
|
|
}
|
2022-05-07 00:59:57 +03:00
|
|
|
}
|
|
|
|
return aff_osds;
|
|
|
|
}
|
|
|
|
|
2024-04-07 00:38:22 +03:00
|
|
|
get_pg_rules(pool_id, pool_cfg, warn)
|
|
|
|
{
|
|
|
|
if (pool_cfg.level_placement)
|
|
|
|
{
|
|
|
|
const pg_size = (0|pool_cfg.pg_size);
|
|
|
|
let rules = pool_cfg.level_placement;
|
|
|
|
if (typeof rules === 'string')
|
|
|
|
{
|
|
|
|
rules = rules.split(/\s+/).map(s => s.split(/=/, 2)).reduce((a, c) => { a[c[0]] = c[1]; return a; }, {});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
rules = { ...rules };
|
|
|
|
}
|
|
|
|
// Always add failure_domain to prevent rules from being totally incorrect
|
|
|
|
const all_diff = [];
|
|
|
|
for (let i = 1; i <= pg_size; i++)
|
|
|
|
{
|
|
|
|
all_diff.push(i);
|
|
|
|
}
|
|
|
|
rules[pool_cfg.failure_domain || 'host'] = all_diff;
|
|
|
|
const levels = this.config.placement_levels||{};
|
|
|
|
levels.host = levels.host || 100;
|
|
|
|
levels.osd = levels.osd || 101;
|
|
|
|
for (const k in rules)
|
|
|
|
{
|
|
|
|
if (!levels[k] || typeof rules[k] !== 'string' &&
|
2024-04-16 02:38:32 +03:00
|
|
|
(!(rules[k] instanceof Array) ||
|
2024-04-07 00:38:22 +03:00
|
|
|
rules[k].filter(s => typeof s !== 'string' && typeof s !== 'number').length > 0))
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' configuration is invalid: level_placement should be { [level]: string | (string|number)[] }');
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
else if (rules[k].length != pg_size)
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' configuration is invalid: values in level_placement should contain exactly pg_size ('+pg_size+') items');
|
|
|
|
return null;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return parse_level_indexes(rules);
|
|
|
|
}
|
|
|
|
else if (typeof pool_cfg.raw_placement === 'string')
|
|
|
|
{
|
|
|
|
try
|
|
|
|
{
|
|
|
|
return parse_pg_dsl(pool_cfg.raw_placement);
|
|
|
|
}
|
|
|
|
catch (e)
|
|
|
|
{
|
|
|
|
if (warn)
|
|
|
|
console.log('Pool '+pool_id+' configuration is invalid: invalid raw_placement: '+e.message);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
let rules = [ [] ];
|
|
|
|
let prev = [ 1 ];
|
|
|
|
for (let i = 1; i < pool_cfg.pg_size; i++)
|
|
|
|
{
|
|
|
|
rules.push([ [ pool_cfg.failure_domain||'host', '!=', prev ] ]);
|
|
|
|
prev = [ ...prev, i+1 ];
|
|
|
|
}
|
|
|
|
return rules;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-28 01:50:26 +03:00
|
|
|
async generate_pool_pgs(pool_id, osd_tree, levels)
|
|
|
|
{
|
|
|
|
const pool_cfg = this.state.config.pools[pool_id];
|
|
|
|
if (!this.validate_pool_cfg(pool_id, pool_cfg, false))
|
|
|
|
{
|
|
|
|
return null;
|
|
|
|
}
|
2024-04-09 15:48:31 +03:00
|
|
|
let pool_tree = { ...osd_tree };
|
2024-01-01 18:40:41 +03:00
|
|
|
this.filter_osds_by_root_node(pool_tree, pool_cfg.root_node);
|
|
|
|
this.filter_osds_by_tags(pool_tree, pool_cfg.osd_tags);
|
2023-12-28 01:50:26 +03:00
|
|
|
this.filter_osds_by_block_layout(
|
|
|
|
pool_tree,
|
|
|
|
pool_cfg.block_size || this.config.block_size || 131072,
|
|
|
|
pool_cfg.bitmap_granularity || this.config.bitmap_granularity || 4096,
|
|
|
|
pool_cfg.immediate_commit || this.config.immediate_commit || 'none'
|
|
|
|
);
|
2024-04-09 15:58:32 +03:00
|
|
|
pool_tree = this.make_hier_tree(pool_tree);
|
2023-12-28 01:50:26 +03:00
|
|
|
// First try last_clean_pgs to minimize data movement
|
|
|
|
let prev_pgs = [];
|
|
|
|
for (const pg in ((this.state.history.last_clean_pgs.items||{})[pool_id]||{}))
|
|
|
|
{
|
|
|
|
prev_pgs[pg-1] = [ ...this.state.history.last_clean_pgs.items[pool_id][pg].osd_set ];
|
|
|
|
}
|
|
|
|
if (!prev_pgs.length)
|
|
|
|
{
|
|
|
|
// Fall back to config/pgs if it's empty
|
|
|
|
for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{}))
|
|
|
|
{
|
|
|
|
prev_pgs[pg-1] = [ ...this.state.config.pgs.items[pool_id][pg].osd_set ];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const old_pg_count = prev_pgs.length;
|
|
|
|
const optimize_cfg = {
|
2024-01-01 18:40:41 +03:00
|
|
|
osd_weights: Object.values(pool_tree).filter(item => item.level === 'osd').reduce((a, c) => { a[c.id] = c.size; return a; }, {}),
|
2024-04-07 00:38:22 +03:00
|
|
|
combinator: !this.config.use_old_pg_combinator || pool_cfg.level_placement || pool_cfg.raw_placement
|
|
|
|
// new algorithm:
|
2024-04-09 15:48:31 +03:00
|
|
|
? new RuleCombinator(pool_tree, this.get_pg_rules(pool_id, pool_cfg), pool_cfg.max_osd_combinations)
|
2024-04-07 00:38:22 +03:00
|
|
|
// old algorithm:
|
2024-04-09 15:48:31 +03:00
|
|
|
: new SimpleCombinator(flatten_tree(pool_tree[''].children, levels, pool_cfg.failure_domain, 'osd'), pool_cfg.pg_size, pool_cfg.max_osd_combinations),
|
2023-12-28 01:50:26 +03:00
|
|
|
pg_count: pool_cfg.pg_count,
|
|
|
|
pg_size: pool_cfg.pg_size,
|
|
|
|
pg_minsize: pool_cfg.pg_minsize,
|
|
|
|
ordered: pool_cfg.scheme != 'replicated',
|
|
|
|
};
|
|
|
|
let optimize_result;
|
|
|
|
// Re-shuffle PGs if config/pgs.hash is empty
|
|
|
|
if (old_pg_count > 0 && this.state.config.pgs.hash)
|
|
|
|
{
|
|
|
|
if (prev_pgs.length != pool_cfg.pg_count)
|
|
|
|
{
|
|
|
|
// Scale PG count
|
|
|
|
// Do it even if old_pg_count is already equal to pool_cfg.pg_count,
|
|
|
|
// because last_clean_pgs may still contain the old number of PGs
|
|
|
|
PGUtil.scale_pg_count(prev_pgs, pool_cfg.pg_count);
|
|
|
|
}
|
|
|
|
for (const pg of prev_pgs)
|
|
|
|
{
|
|
|
|
while (pg.length < pool_cfg.pg_size)
|
|
|
|
{
|
|
|
|
pg.push(0);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
optimize_result = await LPOptimizer.optimize_change({
|
|
|
|
prev_pgs,
|
|
|
|
...optimize_cfg,
|
|
|
|
});
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
optimize_result = await LPOptimizer.optimize_initial(optimize_cfg);
|
|
|
|
}
|
|
|
|
console.log(`Pool ${pool_id} (${pool_cfg.name || 'unnamed'}):`);
|
|
|
|
LPOptimizer.print_change_stats(optimize_result);
|
2024-04-16 02:20:18 +03:00
|
|
|
let pg_effsize = pool_cfg.pg_size;
|
|
|
|
for (const pg of optimize_result.int_pgs)
|
|
|
|
{
|
|
|
|
const this_pg_size = pg.filter(osd => osd != LPOptimizer.NO_OSD).length;
|
|
|
|
if (this_pg_size && this_pg_size < pg_effsize)
|
|
|
|
{
|
|
|
|
pg_effsize = this_pg_size;
|
|
|
|
}
|
|
|
|
}
|
2023-12-28 01:50:26 +03:00
|
|
|
return {
|
|
|
|
pool_id,
|
|
|
|
pgs: optimize_result.int_pgs,
|
|
|
|
stats: {
|
|
|
|
total_raw_tb: optimize_result.space,
|
|
|
|
pg_real_size: pg_effsize || pool_cfg.pg_size,
|
|
|
|
raw_to_usable: (pg_effsize || pool_cfg.pg_size) / (pool_cfg.scheme === 'replicated'
|
|
|
|
? 1 : (pool_cfg.pg_size - (pool_cfg.parity_chunks||0))),
|
|
|
|
space_efficiency: optimize_result.space/(optimize_result.total_space||1),
|
|
|
|
},
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
2020-05-14 19:29:35 +03:00
|
|
|
async recheck_pgs()
|
|
|
|
{
|
2023-11-04 20:59:56 +03:00
|
|
|
if (this.recheck_pgs_active)
|
|
|
|
{
|
|
|
|
this.schedule_recheck();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
this.recheck_pgs_active = true;
|
2020-05-14 19:29:35 +03:00
|
|
|
// Take configuration and state, check it against the stored configuration hash
|
|
|
|
// Recalculate PGs and save them to etcd if the configuration is changed
|
2021-04-03 23:39:25 +03:00
|
|
|
// FIXME: Do not change anything if the distribution is good and random enough and no PGs are degraded
|
2020-10-04 17:20:09 +03:00
|
|
|
const { up_osds, levels, osd_tree } = this.get_osd_tree();
|
2020-05-14 19:29:35 +03:00
|
|
|
const tree_cfg = {
|
2020-09-13 12:23:11 +03:00
|
|
|
osd_tree,
|
2023-12-28 01:50:26 +03:00
|
|
|
levels,
|
2020-09-01 18:50:23 +03:00
|
|
|
pools: this.state.config.pools,
|
2020-05-14 19:29:35 +03:00
|
|
|
};
|
|
|
|
const tree_hash = sha1hex(stableStringify(tree_cfg));
|
|
|
|
if (this.state.config.pgs.hash != tree_hash)
|
|
|
|
{
|
|
|
|
// Something has changed
|
2023-12-28 01:50:26 +03:00
|
|
|
console.log('Pool configuration or OSD tree changed, re-optimizing');
|
|
|
|
// First re-optimize PGs, but don't look at history yet
|
2024-01-02 00:50:23 +03:00
|
|
|
const optimize_results = (await Promise.all(Object.keys(this.state.config.pools)
|
|
|
|
.map(pool_id => this.generate_pool_pgs(pool_id, osd_tree, levels)))).filter(r => r);
|
2023-12-28 01:50:26 +03:00
|
|
|
// Then apply the modification in the form of an optimistic transaction,
|
|
|
|
// each time considering new pg/history modifications (OSDs modify it during rebalance)
|
|
|
|
while (!await this.apply_pool_pgs(optimize_results, up_osds, osd_tree, tree_hash))
|
2020-12-01 14:13:24 +03:00
|
|
|
{
|
2023-12-28 01:50:26 +03:00
|
|
|
console.log(
|
|
|
|
'Someone changed PG configuration while we also tried to change it.'+
|
|
|
|
' Retrying in '+this.config.mon_retry_change_timeout+' ms'
|
2023-09-02 17:55:53 +03:00
|
|
|
);
|
2023-12-28 01:50:26 +03:00
|
|
|
// Failed to apply - parallel change detected. Wait a bit and retry
|
|
|
|
const old_rev = this.etcd_watch_revision;
|
|
|
|
while (this.etcd_watch_revision === old_rev)
|
2021-03-10 01:13:05 +03:00
|
|
|
{
|
2023-12-28 01:50:26 +03:00
|
|
|
await new Promise(ok => setTimeout(ok, this.config.mon_retry_change_timeout));
|
2021-03-10 01:13:05 +03:00
|
|
|
}
|
2023-12-28 01:50:26 +03:00
|
|
|
const new_ot = this.get_osd_tree();
|
|
|
|
const new_tcfg = {
|
|
|
|
osd_tree: new_ot.osd_tree,
|
|
|
|
levels: new_ot.levels,
|
|
|
|
pools: this.state.config.pools,
|
2021-04-03 23:39:25 +03:00
|
|
|
};
|
2023-12-28 01:50:26 +03:00
|
|
|
if (sha1hex(stableStringify(new_tcfg)) !== tree_hash)
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2023-12-28 01:50:26 +03:00
|
|
|
// Configuration actually changed, restart from the beginning
|
|
|
|
this.recheck_pgs_active = false;
|
|
|
|
setImmediate(() => this.recheck_pgs().catch(this.die));
|
|
|
|
return;
|
2020-09-01 18:50:23 +03:00
|
|
|
}
|
2023-12-28 01:50:26 +03:00
|
|
|
// Configuration didn't change, PG history probably changed, so just retry
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
2023-12-28 01:50:26 +03:00
|
|
|
console.log('PG configuration successfully changed');
|
2020-09-13 12:23:11 +03:00
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
2021-03-14 01:26:06 +03:00
|
|
|
// Nothing changed, but we still want to recheck the distribution of primaries
|
2023-05-07 23:17:51 +03:00
|
|
|
let new_config_pgs;
|
2020-09-13 12:23:11 +03:00
|
|
|
let changed = false;
|
|
|
|
for (const pool_id in this.state.config.pools)
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
2020-09-13 12:23:11 +03:00
|
|
|
const pool_cfg = this.state.config.pools[pool_id];
|
|
|
|
if (!this.validate_pool_cfg(pool_id, pool_cfg, false))
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2022-05-07 00:59:57 +03:00
|
|
|
const aff_osds = this.get_affinity_osds(pool_cfg, up_osds, osd_tree);
|
2021-03-14 01:26:06 +03:00
|
|
|
this.reset_rng();
|
|
|
|
for (let pg_num = 1; pg_num <= pool_cfg.pg_count; pg_num++)
|
2020-09-13 12:23:11 +03:00
|
|
|
{
|
2024-04-20 02:02:47 +03:00
|
|
|
if (!this.state.config.pgs.items[pool_id])
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2020-10-17 02:28:48 +03:00
|
|
|
const pg_cfg = this.state.config.pgs.items[pool_id][pg_num];
|
2021-03-14 01:26:06 +03:00
|
|
|
if (pg_cfg)
|
2020-09-13 12:23:11 +03:00
|
|
|
{
|
2022-05-07 00:59:57 +03:00
|
|
|
const new_primary = this.pick_primary(pool_id, pg_cfg.osd_set, up_osds, aff_osds);
|
2020-09-13 12:23:11 +03:00
|
|
|
if (pg_cfg.primary != new_primary)
|
|
|
|
{
|
2023-05-07 23:17:51 +03:00
|
|
|
if (!new_config_pgs)
|
|
|
|
{
|
|
|
|
new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
|
|
|
|
}
|
2020-09-13 12:23:11 +03:00
|
|
|
console.log(
|
|
|
|
`Moving pool ${pool_id} (${pool_cfg.name || 'unnamed'}) PG ${pg_num}`+
|
|
|
|
` primary OSD from ${pg_cfg.primary} to ${new_primary}`
|
|
|
|
);
|
|
|
|
changed = true;
|
2023-05-07 23:17:51 +03:00
|
|
|
new_config_pgs.items[pool_id][pg_num].primary = new_primary;
|
2020-09-13 12:23:11 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
if (changed)
|
|
|
|
{
|
2024-01-16 23:49:21 +03:00
|
|
|
const ok = await this.save_pg_config(new_config_pgs);
|
|
|
|
if (ok)
|
|
|
|
console.log('PG configuration successfully changed');
|
|
|
|
else
|
|
|
|
{
|
|
|
|
console.log('Someone changed PG configuration while we also tried to change it. Retrying in '+this.config.mon_change_timeout+' ms');
|
|
|
|
this.schedule_recheck();
|
|
|
|
}
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2023-11-04 20:59:56 +03:00
|
|
|
this.recheck_pgs_active = false;
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
2023-12-28 01:50:26 +03:00
|
|
|
async apply_pool_pgs(results, up_osds, osd_tree, tree_hash)
|
2020-09-13 12:23:11 +03:00
|
|
|
{
|
2023-12-28 01:50:26 +03:00
|
|
|
for (const pool_id in (this.state.config.pgs||{}).items||{})
|
|
|
|
{
|
|
|
|
// We should stop all PGs when deleting a pool or changing its PG count
|
|
|
|
if (!this.state.config.pools[pool_id] ||
|
|
|
|
this.state.config.pgs.items[pool_id] && this.state.config.pools[pool_id].pg_count !=
|
|
|
|
Object.keys(this.state.config.pgs.items[pool_id]).reduce((a, c) => (a < (0|c) ? (0|c) : a), 0))
|
|
|
|
{
|
|
|
|
if (!await this.stop_all_pgs(pool_id))
|
|
|
|
{
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const new_config_pgs = JSON.parse(JSON.stringify(this.state.config.pgs));
|
|
|
|
const etcd_request = { compare: [], success: [] };
|
|
|
|
for (const pool_id in (new_config_pgs||{}).items||{})
|
|
|
|
{
|
|
|
|
if (!this.state.config.pools[pool_id])
|
|
|
|
{
|
|
|
|
const prev_pgs = [];
|
|
|
|
for (const pg in new_config_pgs.items[pool_id]||{})
|
|
|
|
{
|
|
|
|
prev_pgs[pg-1] = new_config_pgs.items[pool_id][pg].osd_set;
|
|
|
|
}
|
|
|
|
// Also delete pool statistics
|
|
|
|
etcd_request.success.push({ requestDeleteRange: {
|
|
|
|
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
|
|
|
|
} });
|
|
|
|
this.save_new_pgs_txn(new_config_pgs, etcd_request, pool_id, up_osds, osd_tree, prev_pgs, [], []);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
for (const pool_res of results)
|
|
|
|
{
|
|
|
|
const pool_id = pool_res.pool_id;
|
|
|
|
const pool_cfg = this.state.config.pools[pool_id];
|
|
|
|
let pg_history = [];
|
|
|
|
for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{}))
|
|
|
|
{
|
|
|
|
if (this.state.pg.history[pool_id] &&
|
|
|
|
this.state.pg.history[pool_id][pg])
|
|
|
|
{
|
|
|
|
pg_history[pg-1] = this.state.pg.history[pool_id][pg];
|
|
|
|
}
|
|
|
|
}
|
|
|
|
const real_prev_pgs = [];
|
|
|
|
for (const pg in ((this.state.config.pgs.items||{})[pool_id]||{}))
|
|
|
|
{
|
|
|
|
real_prev_pgs[pg-1] = [ ...this.state.config.pgs.items[pool_id][pg].osd_set ];
|
|
|
|
}
|
|
|
|
if (real_prev_pgs.length > 0 && real_prev_pgs.length != pool_res.pgs.length)
|
|
|
|
{
|
|
|
|
console.log(
|
|
|
|
`Changing PG count for pool ${pool_id} (${pool_cfg.name || 'unnamed'})`+
|
|
|
|
` from: ${real_prev_pgs.length} to ${pool_res.pgs.length}`
|
|
|
|
);
|
|
|
|
pg_history = PGUtil.scale_pg_history(pg_history, real_prev_pgs, pool_res.pgs);
|
|
|
|
// Drop stats
|
|
|
|
etcd_request.success.push({ requestDeleteRange: {
|
|
|
|
key: b64(this.etcd_prefix+'/pg/stats/'+pool_id+'/'),
|
|
|
|
range_end: b64(this.etcd_prefix+'/pg/stats/'+pool_id+'0'),
|
|
|
|
} });
|
|
|
|
}
|
|
|
|
const stats = {
|
|
|
|
used_raw_tb: (this.state.pool.stats[pool_id]||{}).used_raw_tb || 0,
|
|
|
|
...pool_res.stats,
|
|
|
|
};
|
|
|
|
etcd_request.success.push({ requestPut: {
|
|
|
|
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
|
|
|
|
value: b64(JSON.stringify(stats)),
|
|
|
|
} });
|
|
|
|
this.save_new_pgs_txn(new_config_pgs, etcd_request, pool_id, up_osds, osd_tree, real_prev_pgs, pool_res.pgs, pg_history);
|
|
|
|
}
|
|
|
|
new_config_pgs.hash = tree_hash;
|
2024-01-16 23:49:21 +03:00
|
|
|
return await this.save_pg_config(new_config_pgs, etcd_request);
|
|
|
|
}
|
|
|
|
|
|
|
|
async save_pg_config(new_config_pgs, etcd_request = { compare: [], success: [] })
|
|
|
|
{
|
2020-09-13 12:23:11 +03:00
|
|
|
etcd_request.compare.push(
|
|
|
|
{ key: b64(this.etcd_prefix+'/mon/master'), target: 'LEASE', lease: ''+this.etcd_lease_id },
|
|
|
|
{ key: b64(this.etcd_prefix+'/config/pgs'), target: 'MOD', mod_revision: ''+this.etcd_watch_revision, result: 'LESS' },
|
|
|
|
);
|
|
|
|
etcd_request.success.push(
|
2023-05-07 23:17:51 +03:00
|
|
|
{ requestPut: { key: b64(this.etcd_prefix+'/config/pgs'), value: b64(JSON.stringify(new_config_pgs)) } },
|
2020-09-13 12:23:11 +03:00
|
|
|
);
|
2023-12-28 01:50:26 +03:00
|
|
|
const txn_res = await this.etcd_call('/kv/txn', etcd_request, this.config.etcd_mon_timeout, 0);
|
|
|
|
return txn_res.succeeded;
|
2020-09-13 12:23:11 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
// Schedule next recheck at least at <unixtime>
|
|
|
|
schedule_next_recheck_at(unixtime)
|
|
|
|
{
|
|
|
|
this.next_recheck_at = !this.next_recheck_at || this.next_recheck_at > unixtime
|
|
|
|
? unixtime : this.next_recheck_at;
|
|
|
|
const now = Date.now()/1000;
|
|
|
|
if (this.next_recheck_timer)
|
|
|
|
{
|
|
|
|
clearTimeout(this.next_recheck_timer);
|
|
|
|
this.next_recheck_timer = null;
|
|
|
|
}
|
|
|
|
if (this.next_recheck_at < now)
|
|
|
|
{
|
|
|
|
this.next_recheck_at = 0;
|
|
|
|
this.schedule_recheck();
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
this.next_recheck_timer = setTimeout(() =>
|
|
|
|
{
|
|
|
|
this.next_recheck_timer = null;
|
|
|
|
this.next_recheck_at = 0;
|
|
|
|
this.schedule_recheck();
|
|
|
|
}, now-this.next_recheck_at);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// Schedule a recheck to run after a small timeout (1s)
|
|
|
|
// This is required for multiple change events to trigger at most 1 recheck in 1s
|
2020-05-14 19:29:35 +03:00
|
|
|
schedule_recheck()
|
|
|
|
{
|
2023-01-05 13:48:06 +03:00
|
|
|
if (!this.recheck_timer)
|
2020-05-14 19:29:35 +03:00
|
|
|
{
|
2023-01-05 13:48:06 +03:00
|
|
|
this.recheck_timer = setTimeout(() =>
|
|
|
|
{
|
|
|
|
this.recheck_timer = null;
|
|
|
|
this.recheck_pgs().catch(this.die);
|
|
|
|
}, this.config.mon_change_timeout || 1000);
|
2020-05-14 19:29:35 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-10-29 01:26:32 +03:00
|
|
|
derive_osd_stats(st, prev, prev_diff)
|
2023-06-18 00:15:39 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
const diff = { op_stats: {}, subop_stats: {}, recovery_stats: {}, inode_stats: {} };
|
2023-11-24 01:05:21 +03:00
|
|
|
if (!st || !st.time || !prev || !prev.time || prev.time >= st.time)
|
2023-06-18 00:15:39 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
return prev_diff || diff;
|
2023-06-18 00:15:39 +03:00
|
|
|
}
|
2023-10-29 01:26:32 +03:00
|
|
|
const timediff = BigInt(st.time*1000 - prev.time*1000);
|
2023-06-18 00:15:39 +03:00
|
|
|
for (const op in st.op_stats||{})
|
|
|
|
{
|
|
|
|
const pr = prev && prev.op_stats && prev.op_stats[op];
|
|
|
|
let c = st.op_stats[op];
|
|
|
|
c = { bytes: BigInt(c.bytes||0), usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
|
|
|
|
const b = c.bytes - BigInt(pr && pr.bytes||0);
|
|
|
|
const us = c.usec - BigInt(pr && pr.usec||0);
|
|
|
|
const n = c.count - BigInt(pr && pr.count||0);
|
|
|
|
if (n > 0)
|
|
|
|
diff.op_stats[op] = { ...c, bps: b*1000n/timediff, iops: n*1000n/timediff, lat: us/n };
|
|
|
|
}
|
|
|
|
for (const op in st.subop_stats||{})
|
|
|
|
{
|
|
|
|
const pr = prev && prev.subop_stats && prev.subop_stats[op];
|
|
|
|
let c = st.subop_stats[op];
|
|
|
|
c = { usec: BigInt(c.usec||0), count: BigInt(c.count||0) };
|
|
|
|
const us = c.usec - BigInt(pr && pr.usec||0);
|
|
|
|
const n = c.count - BigInt(pr && pr.count||0);
|
|
|
|
if (n > 0)
|
|
|
|
diff.subop_stats[op] = { ...c, iops: n*1000n/timediff, lat: us/n };
|
|
|
|
}
|
|
|
|
for (const op in st.recovery_stats||{})
|
|
|
|
{
|
|
|
|
const pr = prev && prev.recovery_stats && prev.recovery_stats[op];
|
|
|
|
let c = st.recovery_stats[op];
|
|
|
|
c = { bytes: BigInt(c.bytes||0), count: BigInt(c.count||0) };
|
|
|
|
const b = c.bytes - BigInt(pr && pr.bytes||0);
|
|
|
|
const n = c.count - BigInt(pr && pr.count||0);
|
|
|
|
if (n > 0)
|
|
|
|
diff.recovery_stats[op] = { ...c, bps: b*1000n/timediff, iops: n*1000n/timediff };
|
|
|
|
}
|
2023-10-29 01:26:32 +03:00
|
|
|
for (const pool_id in st.inode_stats||{})
|
|
|
|
{
|
2024-04-20 02:02:13 +03:00
|
|
|
diff.inode_stats[pool_id] = {};
|
2023-10-29 01:26:32 +03:00
|
|
|
for (const inode_num in st.inode_stats[pool_id])
|
|
|
|
{
|
|
|
|
const inode_diff = diff.inode_stats[pool_id][inode_num] = {};
|
|
|
|
for (const op of [ 'read', 'write', 'delete' ])
|
|
|
|
{
|
|
|
|
const c = st.inode_stats[pool_id][inode_num][op];
|
|
|
|
const pr = prev && prev.inode_stats && prev.inode_stats[pool_id] &&
|
|
|
|
prev.inode_stats[pool_id][inode_num] && prev.inode_stats[pool_id][inode_num][op];
|
|
|
|
const n = BigInt(c.count||0) - BigInt(pr && pr.count||0);
|
|
|
|
inode_diff[op] = {
|
|
|
|
bps: (BigInt(c.bytes||0) - BigInt(pr && pr.bytes||0))*1000n/timediff,
|
|
|
|
iops: n*1000n/timediff,
|
|
|
|
lat: (BigInt(c.usec||0) - BigInt(pr && pr.usec||0))/(n || 1n),
|
|
|
|
};
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2023-06-18 00:15:39 +03:00
|
|
|
return diff;
|
|
|
|
}
|
|
|
|
|
2023-10-29 01:26:32 +03:00
|
|
|
sum_op_stats()
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
for (const osd in this.state.osd.stats)
|
2021-11-10 23:51:15 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
const cur = { ...this.state.osd.stats[osd], inode_stats: this.state.osd.inodestats[osd]||{} };
|
|
|
|
this.prev_stats.osd_diff[osd] = this.derive_osd_stats(
|
|
|
|
cur, this.prev_stats.osd_stats[osd], this.prev_stats.osd_diff[osd]
|
|
|
|
);
|
|
|
|
this.prev_stats.osd_stats[osd] = cur;
|
2021-11-10 23:51:15 +03:00
|
|
|
}
|
2023-10-29 01:26:32 +03:00
|
|
|
const sum_diff = { op_stats: {}, subop_stats: {}, recovery_stats: {} };
|
2023-06-18 00:15:39 +03:00
|
|
|
// Sum derived values instead of deriving summed
|
2024-02-08 21:28:03 +03:00
|
|
|
for (const osd in this.state.osd.state)
|
2021-11-10 23:51:15 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
const derived = this.prev_stats.osd_diff[osd];
|
2024-02-08 21:28:03 +03:00
|
|
|
if (!this.state.osd.state[osd] || !derived)
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2023-10-29 01:26:32 +03:00
|
|
|
for (const type in sum_diff)
|
2022-02-11 16:37:16 +03:00
|
|
|
{
|
2023-10-29 01:26:32 +03:00
|
|
|
for (const op in derived[type]||{})
|
2023-06-18 00:15:39 +03:00
|
|
|
{
|
|
|
|
for (const k in derived[type][op])
|
|
|
|
{
|
|
|
|
sum_diff[type][op] = sum_diff[type][op] || {};
|
|
|
|
sum_diff[type][op][k] = (sum_diff[type][op][k] || 0n) + derived[type][op][k];
|
|
|
|
}
|
|
|
|
}
|
2022-02-11 16:37:16 +03:00
|
|
|
}
|
2021-11-10 23:51:15 +03:00
|
|
|
}
|
2023-06-18 00:15:39 +03:00
|
|
|
return sum_diff;
|
2021-01-21 00:30:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
sum_object_counts()
|
|
|
|
{
|
2020-05-15 01:28:44 +03:00
|
|
|
const object_counts = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
|
2022-08-09 02:27:02 +03:00
|
|
|
const object_bytes = { object: 0n, clean: 0n, misplaced: 0n, degraded: 0n, incomplete: 0n };
|
2020-09-01 18:50:23 +03:00
|
|
|
for (const pool_id in this.state.pg.stats)
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2022-08-09 02:27:02 +03:00
|
|
|
let object_size = 0;
|
|
|
|
for (const osd_num of this.state.pg.stats[pool_id].write_osd_set||[])
|
|
|
|
{
|
|
|
|
if (osd_num && this.state.osd.stats[osd_num] && this.state.osd.stats[osd_num].block_size)
|
|
|
|
{
|
|
|
|
object_size = this.state.osd.stats[osd_num].block_size;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
2023-07-27 05:25:35 +03:00
|
|
|
const pool_cfg = (this.state.config.pools[pool_id]||{});
|
2022-08-09 02:27:02 +03:00
|
|
|
if (!object_size)
|
|
|
|
{
|
2023-07-27 05:25:35 +03:00
|
|
|
object_size = pool_cfg.block_size || this.config.block_size || 131072;
|
|
|
|
}
|
|
|
|
if (pool_cfg.scheme !== 'replicated')
|
|
|
|
{
|
|
|
|
object_size *= ((pool_cfg.pg_size||0) - (pool_cfg.parity_chunks||0));
|
2022-08-09 02:27:02 +03:00
|
|
|
}
|
|
|
|
object_size = BigInt(object_size);
|
2020-09-01 18:50:23 +03:00
|
|
|
for (const pg_num in this.state.pg.stats[pool_id])
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2020-09-01 18:50:23 +03:00
|
|
|
const st = this.state.pg.stats[pool_id][pg_num];
|
2021-03-08 23:13:08 +03:00
|
|
|
if (st)
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2021-03-08 23:13:08 +03:00
|
|
|
for (const k in object_counts)
|
2020-09-01 18:50:23 +03:00
|
|
|
{
|
2021-03-08 23:13:08 +03:00
|
|
|
if (st[k+'_count'])
|
|
|
|
{
|
|
|
|
object_counts[k] += BigInt(st[k+'_count']);
|
2022-08-09 02:27:02 +03:00
|
|
|
object_bytes[k] += BigInt(st[k+'_count']) * object_size;
|
2021-03-08 23:13:08 +03:00
|
|
|
}
|
2020-09-01 18:50:23 +03:00
|
|
|
}
|
2020-05-15 01:28:44 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2022-08-09 02:27:02 +03:00
|
|
|
return { object_counts, object_bytes };
|
2020-05-15 01:28:44 +03:00
|
|
|
}
|
|
|
|
|
2023-10-29 01:26:32 +03:00
|
|
|
sum_inode_stats()
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2021-01-21 00:30:18 +03:00
|
|
|
const inode_stats = {};
|
|
|
|
const inode_stub = () => ({
|
|
|
|
raw_used: 0n,
|
2023-10-29 01:26:32 +03:00
|
|
|
read: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
|
|
|
write: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
|
|
|
delete: { count: 0n, usec: 0n, bytes: 0n, bps: 0n, iops: 0n, lat: 0n },
|
2021-01-21 00:30:18 +03:00
|
|
|
});
|
2021-10-21 14:00:54 +03:00
|
|
|
const seen_pools = {};
|
2021-07-02 22:47:01 +03:00
|
|
|
for (const pool_id in this.state.config.pools)
|
|
|
|
{
|
2021-10-21 14:00:54 +03:00
|
|
|
seen_pools[pool_id] = true;
|
2021-07-02 22:47:01 +03:00
|
|
|
this.state.pool.stats[pool_id] = this.state.pool.stats[pool_id] || {};
|
|
|
|
this.state.pool.stats[pool_id].used_raw_tb = 0n;
|
|
|
|
}
|
2021-01-21 00:30:18 +03:00
|
|
|
for (const osd_num in this.state.osd.space)
|
|
|
|
{
|
2021-04-04 11:52:11 +03:00
|
|
|
for (const pool_id in this.state.osd.space[osd_num])
|
2021-01-21 00:30:18 +03:00
|
|
|
{
|
2021-10-21 14:00:54 +03:00
|
|
|
this.state.pool.stats[pool_id] = this.state.pool.stats[pool_id] || {};
|
|
|
|
if (!seen_pools[pool_id])
|
|
|
|
{
|
|
|
|
this.state.pool.stats[pool_id].used_raw_tb = 0n;
|
|
|
|
seen_pools[pool_id] = true;
|
|
|
|
}
|
2021-04-04 11:52:11 +03:00
|
|
|
inode_stats[pool_id] = inode_stats[pool_id] || {};
|
|
|
|
for (const inode_num in this.state.osd.space[osd_num][pool_id])
|
|
|
|
{
|
2021-07-02 22:47:01 +03:00
|
|
|
const u = BigInt(this.state.osd.space[osd_num][pool_id][inode_num]||0);
|
2024-03-01 01:42:46 +03:00
|
|
|
if (inode_num)
|
|
|
|
{
|
|
|
|
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
|
|
inode_stats[pool_id][inode_num].raw_used += u;
|
|
|
|
}
|
2021-07-02 22:47:01 +03:00
|
|
|
this.state.pool.stats[pool_id].used_raw_tb += u;
|
2021-04-04 11:52:11 +03:00
|
|
|
}
|
2021-01-21 00:30:18 +03:00
|
|
|
}
|
|
|
|
}
|
2021-10-21 14:00:54 +03:00
|
|
|
for (const pool_id in seen_pools)
|
2021-07-02 22:47:01 +03:00
|
|
|
{
|
|
|
|
const used = this.state.pool.stats[pool_id].used_raw_tb;
|
|
|
|
this.state.pool.stats[pool_id].used_raw_tb = Number(used)/1024/1024/1024/1024;
|
|
|
|
}
|
2024-02-08 21:28:03 +03:00
|
|
|
for (const osd_num in this.state.osd.state)
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2021-01-21 00:30:18 +03:00
|
|
|
const ist = this.state.osd.inodestats[osd_num];
|
2024-02-08 21:28:03 +03:00
|
|
|
if (!ist || !this.state.osd.state[osd_num])
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
2021-04-04 11:52:11 +03:00
|
|
|
for (const pool_id in ist)
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2021-04-04 11:52:11 +03:00
|
|
|
inode_stats[pool_id] = inode_stats[pool_id] || {};
|
|
|
|
for (const inode_num in ist[pool_id])
|
2020-05-15 01:28:44 +03:00
|
|
|
{
|
2021-04-04 11:52:11 +03:00
|
|
|
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
|
|
for (const op of [ 'read', 'write', 'delete' ])
|
|
|
|
{
|
|
|
|
inode_stats[pool_id][inode_num][op].count += BigInt(ist[pool_id][inode_num][op].count||0);
|
|
|
|
inode_stats[pool_id][inode_num][op].usec += BigInt(ist[pool_id][inode_num][op].usec||0);
|
|
|
|
inode_stats[pool_id][inode_num][op].bytes += BigInt(ist[pool_id][inode_num][op].bytes||0);
|
|
|
|
}
|
2021-01-21 00:30:18 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2024-02-08 21:28:03 +03:00
|
|
|
for (const osd in this.state.osd.state)
|
2021-01-21 00:30:18 +03:00
|
|
|
{
|
2024-02-08 21:28:03 +03:00
|
|
|
const osd_diff = this.prev_stats.osd_diff[osd];
|
|
|
|
if (!osd_diff || !this.state.osd.state[osd])
|
|
|
|
{
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
for (const pool_id in osd_diff.inode_stats)
|
2023-10-29 01:26:32 +03:00
|
|
|
{
|
|
|
|
for (const inode_num in this.prev_stats.osd_diff[osd].inode_stats[pool_id])
|
|
|
|
{
|
|
|
|
inode_stats[pool_id][inode_num] = inode_stats[pool_id][inode_num] || inode_stub();
|
|
|
|
for (const op of [ 'read', 'write', 'delete' ])
|
|
|
|
{
|
|
|
|
const op_diff = this.prev_stats.osd_diff[osd].inode_stats[pool_id][inode_num][op] || {};
|
|
|
|
const op_st = inode_stats[pool_id][inode_num][op];
|
|
|
|
op_st.bps += op_diff.bps;
|
|
|
|
op_st.iops += op_diff.iops;
|
|
|
|
op_st.lat += op_diff.lat;
|
|
|
|
op_st.n_osd = (op_st.n_osd || 0) + 1;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-11-10 23:51:15 +03:00
|
|
|
}
|
|
|
|
for (const pool_id in inode_stats)
|
|
|
|
{
|
|
|
|
for (const inode_num in inode_stats[pool_id])
|
2021-01-21 00:30:18 +03:00
|
|
|
{
|
2021-11-28 21:02:05 +03:00
|
|
|
let nonzero = inode_stats[pool_id][inode_num].raw_used > 0;
|
2021-11-10 23:51:15 +03:00
|
|
|
for (const op of [ 'read', 'write', 'delete' ])
|
2021-01-21 00:30:18 +03:00
|
|
|
{
|
2021-11-10 23:51:15 +03:00
|
|
|
const op_st = inode_stats[pool_id][inode_num][op];
|
2023-10-29 01:26:32 +03:00
|
|
|
if (op_st.n_osd)
|
|
|
|
{
|
|
|
|
op_st.lat /= BigInt(op_st.n_osd);
|
|
|
|
delete op_st.n_osd;
|
|
|
|
}
|
|
|
|
if (op_st.bps > 0 || op_st.iops > 0)
|
2021-11-28 21:02:05 +03:00
|
|
|
nonzero = true;
|
|
|
|
}
|
|
|
|
if (!nonzero && (!this.state.config.inode[pool_id] || !this.state.config.inode[pool_id][inode_num]))
|
|
|
|
{
|
|
|
|
// Deleted inode (no data, no I/O, no config)
|
|
|
|
delete inode_stats[pool_id][inode_num];
|
2020-05-15 01:28:44 +03:00
|
|
|
}
|
|
|
|
}
|
2021-01-19 02:14:13 +03:00
|
|
|
}
|
2023-07-06 00:40:13 +03:00
|
|
|
return { inode_stats, seen_pools };
|
2021-01-21 00:30:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
serialize_bigints(obj)
|
|
|
|
{
|
2021-11-10 23:51:15 +03:00
|
|
|
obj = { ...obj };
|
2021-01-21 00:30:18 +03:00
|
|
|
for (const k in obj)
|
2021-01-19 02:14:13 +03:00
|
|
|
{
|
2021-01-21 00:30:18 +03:00
|
|
|
if (typeof obj[k] == 'bigint')
|
|
|
|
{
|
|
|
|
obj[k] = ''+obj[k];
|
|
|
|
}
|
|
|
|
else if (typeof obj[k] == 'object')
|
2021-01-19 02:14:13 +03:00
|
|
|
{
|
2021-11-10 23:51:15 +03:00
|
|
|
obj[k] = this.serialize_bigints(obj[k]);
|
2021-01-19 02:14:13 +03:00
|
|
|
}
|
|
|
|
}
|
2021-11-10 23:51:15 +03:00
|
|
|
return obj;
|
2021-01-21 00:30:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async update_total_stats()
|
|
|
|
{
|
|
|
|
const txn = [];
|
2022-08-09 02:27:02 +03:00
|
|
|
const { object_counts, object_bytes } = this.sum_object_counts();
|
2023-10-29 01:26:32 +03:00
|
|
|
let stats = this.sum_op_stats();
|
|
|
|
let { inode_stats, seen_pools } = this.sum_inode_stats();
|
2021-01-21 00:30:18 +03:00
|
|
|
stats.object_counts = object_counts;
|
2022-08-09 02:27:02 +03:00
|
|
|
stats.object_bytes = object_bytes;
|
2021-11-10 23:51:15 +03:00
|
|
|
stats = this.serialize_bigints(stats);
|
|
|
|
inode_stats = this.serialize_bigints(inode_stats);
|
2021-01-21 00:30:18 +03:00
|
|
|
txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(stats)) } });
|
2021-04-04 11:52:11 +03:00
|
|
|
for (const pool_id in inode_stats)
|
2021-01-19 02:14:13 +03:00
|
|
|
{
|
2021-04-04 11:52:11 +03:00
|
|
|
for (const inode_num in inode_stats[pool_id])
|
|
|
|
{
|
|
|
|
txn.push({ requestPut: {
|
|
|
|
key: b64(this.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num),
|
|
|
|
value: b64(JSON.stringify(inode_stats[pool_id][inode_num])),
|
|
|
|
} });
|
|
|
|
}
|
2021-01-19 02:14:13 +03:00
|
|
|
}
|
2021-11-28 21:02:05 +03:00
|
|
|
for (const pool_id in this.state.inode.stats)
|
|
|
|
{
|
|
|
|
for (const inode_num in this.state.inode.stats[pool_id])
|
|
|
|
{
|
|
|
|
if (!inode_stats[pool_id] || !inode_stats[pool_id][inode_num])
|
|
|
|
{
|
|
|
|
txn.push({ requestDeleteRange: {
|
|
|
|
key: b64(this.etcd_prefix+'/inode/stats/'+pool_id+'/'+inode_num),
|
|
|
|
} });
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2021-07-02 22:47:01 +03:00
|
|
|
for (const pool_id in this.state.pool.stats)
|
|
|
|
{
|
2023-07-06 00:40:13 +03:00
|
|
|
if (!seen_pools[pool_id])
|
|
|
|
{
|
|
|
|
txn.push({ requestDeleteRange: {
|
|
|
|
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
|
|
|
|
} });
|
|
|
|
delete this.state.pool.stats[pool_id];
|
|
|
|
}
|
|
|
|
else
|
|
|
|
{
|
|
|
|
const pool_stats = { ...this.state.pool.stats[pool_id] };
|
|
|
|
this.serialize_bigints(pool_stats);
|
|
|
|
txn.push({ requestPut: {
|
|
|
|
key: b64(this.etcd_prefix+'/pool/stats/'+pool_id),
|
|
|
|
value: b64(JSON.stringify(pool_stats)),
|
|
|
|
} });
|
|
|
|
}
|
2021-07-02 22:47:01 +03:00
|
|
|
}
|
2021-01-19 02:14:13 +03:00
|
|
|
if (txn.length)
|
|
|
|
{
|
|
|
|
await this.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
|
2020-05-15 01:28:44 +03:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
schedule_update_stats()
|
|
|
|
{
|
|
|
|
if (this.stats_timer)
|
|
|
|
{
|
2021-11-09 01:41:22 +03:00
|
|
|
return;
|
2021-07-02 22:47:01 +03:00
|
|
|
}
|
2020-05-15 01:28:44 +03:00
|
|
|
this.stats_timer = setTimeout(() =>
|
|
|
|
{
|
|
|
|
this.stats_timer = null;
|
|
|
|
this.update_total_stats().catch(console.error);
|
2021-11-09 01:41:22 +03:00
|
|
|
}, this.config.mon_stats_timeout);
|
2020-05-15 01:28:44 +03:00
|
|
|
}
|
|
|
|
|
2020-05-12 17:49:18 +03:00
|
|
|
parse_kv(kv)
|
|
|
|
{
|
|
|
|
if (!kv || !kv.key)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
kv.key = de64(kv.key);
|
2020-09-05 02:14:43 +03:00
|
|
|
kv.value = kv.value ? de64(kv.value) : null;
|
|
|
|
let key = kv.key.substr(this.etcd_prefix.length+1);
|
2020-09-01 17:07:06 +03:00
|
|
|
if (!this.constructor.etcd_allow.exec(key))
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
console.log('Bad key in etcd: '+kv.key+' = '+kv.value);
|
|
|
|
return;
|
|
|
|
}
|
2020-09-11 19:25:52 +03:00
|
|
|
try
|
|
|
|
{
|
|
|
|
kv.value = kv.value ? JSON.parse(kv.value) : null;
|
|
|
|
}
|
|
|
|
catch (e)
|
|
|
|
{
|
2020-10-23 01:54:38 +03:00
|
|
|
console.log('Bad value in etcd: '+kv.key+' = '+kv.value);
|
2020-09-11 19:25:52 +03:00
|
|
|
return;
|
|
|
|
}
|
2020-03-12 00:56:41 +03:00
|
|
|
let key_parts = key.split('/');
|
2020-09-01 18:50:23 +03:00
|
|
|
let cur = this.state;
|
2020-03-12 00:56:41 +03:00
|
|
|
for (let i = 0; i < key_parts.length-1; i++)
|
|
|
|
{
|
|
|
|
cur = (cur[key_parts[i]] = cur[key_parts[i]] || {});
|
|
|
|
}
|
|
|
|
if (etcd_nonempty_keys[key])
|
2020-09-01 17:07:06 +03:00
|
|
|
{
|
2020-03-12 00:56:41 +03:00
|
|
|
// Do not clear these to null
|
|
|
|
kv.value = kv.value || {};
|
2020-09-01 17:07:06 +03:00
|
|
|
}
|
2022-12-24 02:47:48 +03:00
|
|
|
const old = cur[key_parts[key_parts.length-1]];
|
2020-03-12 00:56:41 +03:00
|
|
|
cur[key_parts[key_parts.length-1]] = kv.value;
|
|
|
|
if (key === 'config/global')
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
this.config = { ...this.initConfig, ...this.state.config.global };
|
2020-05-14 19:29:35 +03:00
|
|
|
this.check_config();
|
2020-09-13 12:23:11 +03:00
|
|
|
for (const osd_num in this.state.osd.stats)
|
|
|
|
{
|
|
|
|
// Recheck PGs <osd_out_time> later
|
|
|
|
this.schedule_next_recheck_at(
|
|
|
|
!this.state.osd.stats[osd_num] ? 0 : this.state.osd.stats[osd_num].time+this.config.osd_out_time
|
|
|
|
);
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
2020-03-12 00:56:41 +03:00
|
|
|
else if (key === 'config/pools')
|
2020-09-05 02:14:43 +03:00
|
|
|
{
|
|
|
|
for (const pool_id in this.state.config.pools)
|
|
|
|
{
|
2020-09-13 12:23:11 +03:00
|
|
|
// Adjust pool configuration so PG distribution hash doesn't change on recheck()
|
2020-09-05 02:14:43 +03:00
|
|
|
const pool_cfg = this.state.config.pools[pool_id];
|
|
|
|
this.validate_pool_cfg(pool_id, pool_cfg, true);
|
|
|
|
}
|
|
|
|
}
|
2020-03-12 00:56:41 +03:00
|
|
|
else if (key_parts[0] === 'osd' && key_parts[1] === 'stats')
|
2020-09-13 12:23:11 +03:00
|
|
|
{
|
2022-12-24 02:47:48 +03:00
|
|
|
// Recheck OSD tree on OSD addition/deletion
|
2023-06-18 00:15:39 +03:00
|
|
|
const osd_num = key_parts[2];
|
2023-01-05 13:54:10 +03:00
|
|
|
if ((!old) != (!kv.value) || old && kv.value && old.size != kv.value.size)
|
2022-12-24 02:47:48 +03:00
|
|
|
{
|
|
|
|
this.schedule_recheck();
|
|
|
|
}
|
2023-01-05 13:54:10 +03:00
|
|
|
// Recheck PGs <osd_out_time> after last OSD statistics report
|
2020-09-13 12:23:11 +03:00
|
|
|
this.schedule_next_recheck_at(
|
2023-06-18 00:15:39 +03:00
|
|
|
!this.state.osd.stats[osd_num] ? 0 : this.state.osd.stats[osd_num].time+this.config.osd_out_time
|
2020-09-13 12:23:11 +03:00
|
|
|
);
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
|
|
|
async etcd_call(path, body, timeout, retries)
|
|
|
|
{
|
|
|
|
let retry = 0;
|
|
|
|
if (retries >= 0 && retries < 1)
|
|
|
|
{
|
|
|
|
retries = 1;
|
|
|
|
}
|
2023-08-03 23:56:50 +03:00
|
|
|
const tried = {};
|
2020-05-12 17:49:18 +03:00
|
|
|
while (retries < 0 || retry < retries)
|
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
retry++;
|
|
|
|
const base = this.pick_next_etcd();
|
2023-08-03 23:56:50 +03:00
|
|
|
let now = Date.now();
|
|
|
|
if (tried[base] && now-tried[base] < timeout)
|
|
|
|
{
|
|
|
|
await new Promise(ok => setTimeout(ok, timeout-(now-tried[base])));
|
|
|
|
now = Date.now();
|
|
|
|
}
|
|
|
|
tried[base] = now;
|
2020-05-12 17:49:18 +03:00
|
|
|
const res = await POST(base+path, body, timeout);
|
2020-08-03 23:50:50 +03:00
|
|
|
if (res.error)
|
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
if (this.selected_etcd_url == base)
|
|
|
|
this.selected_etcd_url = null;
|
|
|
|
console.error('failed to query etcd: '+res.error);
|
|
|
|
continue;
|
2020-08-03 23:50:50 +03:00
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
if (res.json)
|
|
|
|
{
|
|
|
|
if (res.json.error)
|
|
|
|
{
|
2020-09-05 02:14:43 +03:00
|
|
|
console.error('etcd returned error: '+res.json.error);
|
2020-05-12 17:49:18 +03:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
return res.json;
|
|
|
|
}
|
|
|
|
}
|
2024-01-17 00:16:56 +03:00
|
|
|
this.failconnect();
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
2024-01-17 00:16:56 +03:00
|
|
|
_die(err, code)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
// In fact we can just try to rejoin
|
2024-04-20 02:02:13 +03:00
|
|
|
console.error(err instanceof Error ? err : new Error(err || 'Cluster connection failed'));
|
2024-01-17 00:16:56 +03:00
|
|
|
process.exit(code || 2);
|
2020-05-12 17:49:18 +03:00
|
|
|
}
|
|
|
|
|
2021-11-28 01:19:42 +03:00
|
|
|
local_ips(all)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
const ips = [];
|
|
|
|
const ifaces = os.networkInterfaces();
|
|
|
|
for (const ifname in ifaces)
|
|
|
|
{
|
|
|
|
for (const iface of ifaces[ifname])
|
|
|
|
{
|
2021-11-28 01:19:42 +03:00
|
|
|
if (iface.family == 'IPv4' && !iface.internal || all)
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
ips.push(iface.address);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ips;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
function POST(url, body, timeout)
|
|
|
|
{
|
2024-04-20 02:02:13 +03:00
|
|
|
return new Promise(ok =>
|
2020-05-12 17:49:18 +03:00
|
|
|
{
|
|
|
|
const body_text = Buffer.from(JSON.stringify(body));
|
|
|
|
let timer_id = timeout > 0 ? setTimeout(() =>
|
|
|
|
{
|
|
|
|
if (req)
|
|
|
|
req.abort();
|
|
|
|
req = null;
|
|
|
|
ok({ error: 'timeout' });
|
|
|
|
}, timeout) : null;
|
|
|
|
let req = http.request(url, { method: 'POST', headers: {
|
|
|
|
'Content-Type': 'application/json',
|
2020-08-03 23:50:50 +03:00
|
|
|
'Content-Length': body_text.length,
|
2020-05-12 17:49:18 +03:00
|
|
|
} }, (res) =>
|
|
|
|
{
|
|
|
|
if (!req)
|
|
|
|
{
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
clearTimeout(timer_id);
|
|
|
|
let res_body = '';
|
|
|
|
res.setEncoding('utf8');
|
2023-05-11 02:16:52 +03:00
|
|
|
res.on('error', (error) => ok({ error }));
|
2020-09-01 18:50:23 +03:00
|
|
|
res.on('data', chunk => { res_body += chunk; });
|
2020-05-12 17:49:18 +03:00
|
|
|
res.on('end', () =>
|
|
|
|
{
|
2020-08-03 23:50:50 +03:00
|
|
|
if (res.statusCode != 200)
|
|
|
|
{
|
|
|
|
ok({ error: res_body, code: res.statusCode });
|
|
|
|
return;
|
|
|
|
}
|
2020-05-12 17:49:18 +03:00
|
|
|
try
|
|
|
|
{
|
|
|
|
res_body = JSON.parse(res_body);
|
|
|
|
ok({ response: res, json: res_body });
|
|
|
|
}
|
|
|
|
catch (e)
|
|
|
|
{
|
|
|
|
ok({ error: e, response: res, body: res_body });
|
|
|
|
}
|
|
|
|
});
|
|
|
|
});
|
2023-05-11 02:16:52 +03:00
|
|
|
req.on('error', (error) => ok({ error }));
|
|
|
|
req.on('close', () => ok({ error: new Error('Connection closed prematurely') }));
|
2020-05-12 17:49:18 +03:00
|
|
|
req.write(body_text);
|
|
|
|
req.end();
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
function b64(str)
|
|
|
|
{
|
|
|
|
return Buffer.from(str).toString('base64');
|
|
|
|
}
|
|
|
|
|
|
|
|
function de64(str)
|
|
|
|
{
|
|
|
|
return Buffer.from(str, 'base64').toString();
|
|
|
|
}
|
|
|
|
|
|
|
|
function sha1hex(str)
|
|
|
|
{
|
|
|
|
const hash = crypto.createHash('sha1');
|
|
|
|
hash.update(str);
|
|
|
|
return hash.digest('hex');
|
|
|
|
}
|
2020-09-05 02:14:43 +03:00
|
|
|
|
2020-11-08 01:54:12 +03:00
|
|
|
Mon.etcd_allow = etcd_allow;
|
|
|
|
Mon.etcd_tree = etcd_tree;
|
|
|
|
|
2020-09-05 02:14:43 +03:00
|
|
|
module.exports = Mon;
|