WIP Use random_hier_combinations
Test / buildenv (push) Successful in 10s Details
Test / build (push) Successful in 2m31s Details
Test / test_cas (push) Successful in 10s Details
Test / make_test (push) Successful in 36s Details
Test / test_change_pg_size (push) Successful in 18s Details
Test / test_change_pg_count (push) Successful in 49s Details
Test / test_create_nomaxid (push) Successful in 8s Details
Test / test_failure_domain (push) Successful in 11s Details
Test / test_etcd_fail (push) Successful in 52s Details
Test / test_interrupted_rebalance (push) Successful in 1m21s Details
Test / test_add_osd (push) Successful in 2m35s Details
Test / test_interrupted_rebalance_imm (push) Successful in 1m10s Details
Test / test_minsize_1 (push) Successful in 18s Details
Test / test_interrupted_rebalance_ec_imm (push) Successful in 52s Details
Test / test_move_reappear (push) Successful in 25s Details
Test / test_interrupted_rebalance_ec (push) Successful in 1m33s Details
Test / test_rebalance_verify (push) Successful in 2m37s Details
Test / test_rebalance_verify_imm (push) Successful in 2m34s Details
Test / test_rm (push) Successful in 11s Details
Test / test_rebalance_verify_ec_imm (push) Successful in 2m15s Details
Test / test_snapshot (push) Successful in 20s Details
Test / test_rebalance_verify_ec (push) Successful in 3m6s Details
Test / test_splitbrain (push) Successful in 22s Details
Test / test_snapshot_ec (push) Successful in 32s Details
Test / test_write_no_same (push) Successful in 18s Details
Test / test_write (push) Successful in 55s Details
Test / test_write_xor (push) Successful in 1m41s Details
Test / test_heal_pg_size_2 (push) Successful in 4m21s Details
Test / test_heal_ec (push) Successful in 4m48s Details
Test / test_change_pg_count_ec (push) Successful in 43s Details

hier-failure-domains
Vitaliy Filippov 2023-01-08 00:18:03 +03:00
parent c1d470522c
commit 72f0cff79d
3 changed files with 76 additions and 23 deletions

View File

@ -50,7 +50,8 @@ async function lp_solve(text)
return { score, vars };
}
async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize = 2, hier_sizes = null,
max_combinations = 10000, parity_space = 1, ordered = false, seq_layout = false })
{
if (!pg_count || !osd_tree)
{
@ -58,7 +59,7 @@ async function optimize_initial({ osd_tree, pg_count, pg_size = 3, pg_minsize =
}
const all_weights = Object.assign({}, ...Object.values(osd_tree));
const total_weight = Object.values(all_weights).reduce((a, c) => Number(a) + Number(c), 0);
const all_pgs = Object.values(random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1));
const all_pgs = Object.values(random_hier_combinations(osd_tree, hier_sizes || [ pg_size, 1 ], max_combinations, parity_space > 1, seq_layout));
const pg_per_osd = {};
for (const pg of all_pgs)
{
@ -216,39 +217,45 @@ function calc_intersect_weights(old_pg_size, pg_size, pg_count, prev_weights, al
return move_weights;
}
function add_valid_previous(osd_tree, prev_weights, all_pgs)
function build_parent_per_leaf(osd_tree, res = {}, parents = [])
{
for (const item in osd_tree)
{
if (osd_tree[item] instanceof Object)
build_parent_per_leaf(osd_tree[item], res, [ ...parents, item ]);
else
res[item] = parents;
}
return res;
}
function add_valid_previous(osd_tree, prev_weights, all_pgs, hier_sizes)
{
// Add previous combinations that are still valid
const hosts = Object.keys(osd_tree).sort();
const host_per_osd = {};
for (const host in osd_tree)
{
for (const osd in osd_tree[host])
{
host_per_osd[osd] = host;
}
}
const parent_per_osd = build_parent_per_leaf(osd_tree);
skip_pg: for (const pg_name in prev_weights)
{
const seen_hosts = {};
const seen = [];
const pg = pg_name.substr(3).split(/_/);
for (const osd of pg)
{
if (!host_per_osd[osd] || seen_hosts[host_per_osd[osd]])
{
if (!parent_per_osd[osd])
continue skip_pg;
for (let i = 0; i < parent_per_osd[osd].length; i++)
{
seen[parent_per_osd[osd][i]]++;
if (seen[parent_per_osd[osd][i]] > hier_sizes[i])
continue skip_pg;
}
seen_hosts[host_per_osd[osd]] = true;
}
if (!all_pgs[pg_name])
{
all_pgs[pg_name] = pg;
}
}
}
// Try to minimize data movement
async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2, max_combinations = 10000, parity_space = 1, ordered = false })
async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3, pg_minsize = 2,
hier_sizes = null, max_combinations = 10000, parity_space = 1, ordered = false, seq_layout = false })
{
if (!osd_tree)
{
@ -273,10 +280,10 @@ async function optimize_change({ prev_pgs: prev_int_pgs, osd_tree, pg_size = 3,
}
const old_pg_size = prev_int_pgs[0].length;
// Get all combinations
let all_pgs = random_combinations(osd_tree, pg_size, max_combinations, parity_space > 1);
let all_pgs = random_hier_combinations(osd_tree, hier_sizes || [ pg_size, 1 ], max_combinations, parity_space > 1, seq_layout);
if (old_pg_size == pg_size)
{
add_valid_previous(osd_tree, prev_weights, all_pgs);
add_valid_previous(osd_tree, prev_weights, all_pgs, hier_sizes || [ pg_size, 1 ]);
}
all_pgs = Object.values(all_pgs);
const pg_per_osd = {};

View File

@ -159,6 +159,10 @@ const etcd_tree = {
// number of parity chunks, required for EC
parity_chunks?: 1,
pg_count: 100,
// failure_domain = string | { string: int }
// the second case specifies multiple failure domains. example:
// { datacenter: 3, host: 2 } - means 3 datacenters with 2 hosts each, for EC 4+2
// guarantees availability on outage of either 1 datacenter or 2 hosts
failure_domain: 'host',
max_osd_combinations: 10000,
// block_size, bitmap_granularity, immediate_commit must match all OSDs used in that pool
@ -1027,6 +1031,32 @@ class Mon
pool_cfg.parity_chunks = Math.floor(pool_cfg.parity_chunks) || undefined;
pool_cfg.pg_count = Math.floor(pool_cfg.pg_count);
pool_cfg.failure_domain = pool_cfg.failure_domain || 'host';
if (pool_cfg.failure_domain instanceof Object)
{
for (const key in pool_cfg.failure_domain)
{
const cnt = parseInt(pool_cfg.failure_domain[key]);
if (!cnt || cnt <= 0)
{
if (warn)
console.log('Pool '+pool_id+' specifies invalid item count for failure domain \"'+key+'\"');
return false;
}
if (key !== 'host' && key != 'osd' && !(key in this.config.placement_levels||{}))
{
if (warn)
console.log('Pool '+pool_id+' uses invalid failure domain \"'+key+'\"');
return false;
}
}
}
else if (pool_cfg.failure_domain !== 'host' && pool_cfg.failure_domain != 'osd' &&
!(pool_cfg.failure_domain in this.config.placement_levels||{}))
{
if (warn)
console.log('Pool '+pool_id+' uses invalid failure domain \"'+pool_cfg.failure_domain+'\"');
return false;
}
pool_cfg.max_osd_combinations = Math.floor(pool_cfg.max_osd_combinations) || 10000;
if (!/^[1-9]\d*$/.exec(''+pool_id))
{
@ -1188,7 +1218,10 @@ class Mon
continue;
}
let pool_tree = osd_tree[pool_cfg.root_node || ''] || {};
pool_tree = LPOptimizer.extract_tree_levels(pool_tree, [ pool_cfg.failure_domain, 'osd' ], levels);
const failure_domains = pool_cfg.failure_domain instanceof Object
? [ ...Object.keys(pool_cfg.failure_domain), 'osd' ]
: [ pool_cfg.failure_domain, 'osd' ];
pool_tree = LPOptimizer.extract_tree_levels(pool_tree, failure_domains, levels);
this.filter_osds_by_tags(osd_tree, pool_tree, pool_cfg.osd_tags);
// These are for the purpose of building history.osd_sets
const real_prev_pgs = [];
@ -1215,6 +1248,9 @@ class Mon
pg_count: pool_cfg.pg_count,
pg_size: pool_cfg.pg_size,
pg_minsize: pool_cfg.pg_minsize,
hier_sizes: pool_cfg.failure_domain instanceof Object
? [ ...Object.values(pool_cfg.failure_domain), 1 ]
: null,
max_combinations: pool_cfg.max_osd_combinations,
ordered: pool_cfg.scheme != 'replicated',
};
@ -1270,7 +1306,7 @@ class Mon
} });
}
LPOptimizer.print_change_stats(optimize_result);
const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length);
const pg_effsize = Math.min(pool_cfg.pg_size, Object.keys(pool_tree).length); // FIXME requires hier support too
this.state.pool.stats[pool_id] = {
used_raw_tb: (this.state.pool.stats[pool_id]||{}).used_raw_tb || 0,
total_raw_tb: optimize_result.space,

View File

@ -19,6 +19,14 @@ const osd_tree2 = {
500: { 511: 1, 512: 1, 521: 1, 522: 1 },
};
const osd_tree3 = {
100: { 111: 1, 112: 1, 121: 1, 122: 1 },
200: { 211: 1, 212: 1, 221: 1, 222: 1 },
300: { 311: 1, 312: 1, 321: 1, 322: 1 },
400: { 411: 1, 412: 1, 421: 1, 422: 1 },
500: { 511: 1 },
};
async function run()
{
let r;
@ -40,6 +48,8 @@ async function run()
));
if (JSON.stringify(r) != '{"rack1":{"OSD5":10},"rack2":{"OSD10":10}}')
throw new Error('extract_tree_levels failed');
// should not contain Z:
console.log(r = LPOptimizer.random_hier_combinations(osd_tree3, [ 3, 2 ], 0, false, true));
console.log('OK');
}